From 6b0d9b7a9dcaab9d9ebfadf31845c567868bc1ca Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Wed, 13 Feb 2019 19:17:40 +0000 Subject: [PATCH 001/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 1609ec5ca..b8d3c56e5 100644 --- a/api.go +++ b/api.go @@ -102,7 +102,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.17" + libraryVersion = "v6.0.18" ) // User Agent should always following the below style. From 96136a35dc83992096651b9b7efa47fd2a62b525 Mon Sep 17 00:00:00 2001 From: Savely Krasovsky Date: Mon, 18 Feb 2019 09:16:17 +0300 Subject: [PATCH 002/215] Strip port 80/443 in getBucketLocationRequest (#1069) --- bucket-cache.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/bucket-cache.go b/bucket-cache.go index cac7ad792..67709cae8 100644 --- a/bucket-cache.go +++ b/bucket-cache.go @@ -18,6 +18,7 @@ package minio import ( + "net" "net/http" "net/url" "path" @@ -162,6 +163,14 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro // Set get bucket location always as path style. targetURL := c.endpointURL + + // as it works in makeTargetURL method from api.go file + if h, p, err := net.SplitHostPort(targetURL.Host); err == nil { + if targetURL.Scheme == "http" && p == "80" || targetURL.Scheme == "https" && p == "443" { + targetURL.Host = h + } + } + targetURL.Path = path.Join(bucketName, "") + "/" targetURL.RawQuery = urlValues.Encode() From 76df8feb69649e69c822b765f86591c61a7aad0d Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 19 Feb 2019 11:19:02 -0800 Subject: [PATCH 003/215] Select error-type is now error-code (#1073) --- api-select.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-select.go b/api-select.go index 10e1d47d6..643abef8d 100644 --- a/api-select.go +++ b/api-select.go @@ -325,7 +325,7 @@ func (s *SelectResults) start(pipeWriter *io.PipeWriter) { switch m { case errorMsg: - pipeWriter.CloseWithError(errors.New("Error Type of " + headers.Get("error-type") + " " + headers.Get("error-message"))) + pipeWriter.CloseWithError(errors.New(headers.Get("error-code") + ":\"" + headers.Get("error-message") + "\"")) closeResponse(s.resp) return case commonMsg: From 008c7aa71fc17e11bf980c209a4f8c4d687fc884 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sat, 23 Feb 2019 00:46:38 -0800 Subject: [PATCH 004/215] Support HTTP 2.0 in DefaultTransport for TLS requests (#1077) This fixes the unit tests failure in the master branch --- api-put-object-common.go | 33 +++++++++---------- api.go | 7 +++- go.mod | 2 +- go.sum | 4 +-- transport.go | 70 +++++++++++++++++++++++++++++----------- 5 files changed, 76 insertions(+), 40 deletions(-) diff --git a/api-put-object-common.go b/api-put-object-common.go index c16c3c69a..ce3396e7d 100644 --- a/api-put-object-common.go +++ b/api-put-object-common.go @@ -34,26 +34,25 @@ func isObject(reader io.Reader) (ok bool) { // Verify if reader is a generic ReaderAt func isReadAt(reader io.Reader) (ok bool) { - _, ok = reader.(io.ReaderAt) + var v *os.File + v, ok = reader.(*os.File) if ok { - var v *os.File - v, ok = reader.(*os.File) - if ok { - // Stdin, Stdout and Stderr all have *os.File type - // which happen to also be io.ReaderAt compatible - // we need to add special conditions for them to - // be ignored by this function. - for _, f := range []string{ - "/dev/stdin", - "/dev/stdout", - "/dev/stderr", - } { - if f == v.Name() { - ok = false - break - } + // Stdin, Stdout and Stderr all have *os.File type + // which happen to also be io.ReaderAt compatible + // we need to add special conditions for them to + // be ignored by this function. + for _, f := range []string{ + "/dev/stdin", + "/dev/stdout", + "/dev/stderr", + } { + if f == v.Name() { + ok = false + break } } + } else { + _, ok = reader.(io.ReaderAt) } return } diff --git a/api.go b/api.go index b8d3c56e5..1e77e1200 100644 --- a/api.go +++ b/api.go @@ -295,10 +295,15 @@ func privateNew(endpoint string, creds *credentials.Credentials, secure bool, re // Save endpoint URL, user agent for future uses. clnt.endpointURL = endpointURL + transport, err := DefaultTransport(secure) + if err != nil { + return nil, err + } + // Instantiate http client and bucket location cache. clnt.httpClient = &http.Client{ Jar: jar, - Transport: DefaultTransport, + Transport: transport, CheckRedirect: clnt.redirectHeaders, } diff --git a/go.mod b/go.mod index 9c416086f..59ef0a794 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 // indirect github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c // indirect golang.org/x/crypto v0.0.0-20190128193316-c7b33c32a30b - golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3 + golang.org/x/net v0.0.0-20190213061140-3a22650c66bd golang.org/x/sys v0.0.0-20190124100055-b90733256f2e // indirect golang.org/x/text v0.3.0 // indirect gopkg.in/ini.v1 v1.41.0 diff --git a/go.sum b/go.sum index 5f9d0e6c9..baf2f44e8 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c h1:Ho+uVpke github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= golang.org/x/crypto v0.0.0-20190128193316-c7b33c32a30b h1:Ib/yptP38nXZFMwqWSip+OKuMP9OkyDe3p+DssP8n9w= golang.org/x/crypto v0.0.0-20190128193316-c7b33c32a30b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3 h1:ulvT7fqt0yHWzpJwI57MezWnYDVpCAYBVuYst/L+fAY= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd h1:HuTn7WObtcDo9uEEU7rEqL0jYthdXAmZ6PP+meazmaU= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/sys v0.0.0-20190124100055-b90733256f2e h1:3GIlrlVLfkoipSReOMNAgApI0ajnalyLa/EZHHca/XI= golang.org/x/sys v0.0.0-20190124100055-b90733256f2e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= diff --git a/transport.go b/transport.go index 88700cfe7..d231f8b2a 100644 --- a/transport.go +++ b/transport.go @@ -20,31 +20,63 @@ package minio import ( + "crypto/tls" + "crypto/x509" "net" "net/http" "time" + + "golang.org/x/net/http2" ) // DefaultTransport - this default transport is similar to // http.DefaultTransport but with additional param DisableCompression // is set to true to avoid decompressing content with 'gzip' encoding. -var DefaultTransport http.RoundTripper = &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - }).DialContext, - MaxIdleConns: 100, - MaxIdleConnsPerHost: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - // Set this value so that the underlying transport round-tripper - // doesn't try to auto decode the body of objects with - // content-encoding set to `gzip`. - // - // Refer: - // https://golang.org/src/net/http/transport.go?h=roundTrip#L1843 - DisableCompression: true, +var DefaultTransport = func(secure bool) (http.RoundTripper, error) { + tr := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + MaxIdleConns: 1024, + MaxIdleConnsPerHost: 1024, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + // Set this value so that the underlying transport round-tripper + // doesn't try to auto decode the body of objects with + // content-encoding set to `gzip`. + // + // Refer: + // https://golang.org/src/net/http/transport.go?h=roundTrip#L1843 + DisableCompression: true, + } + + if secure { + rootCAs, _ := x509.SystemCertPool() + if rootCAs == nil { + // In some systems (like Windows) system cert pool is + // not supported or no certificates are present on the + // system - so we create a new cert pool. + rootCAs = x509.NewCertPool() + } + + // Keep TLS config. + tlsConfig := &tls.Config{ + RootCAs: rootCAs, + // Can't use SSLv3 because of POODLE and BEAST + // Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher + // Can't use TLSv1.1 because of RC4 cipher usage + MinVersion: tls.VersionTLS12, + } + tr.TLSClientConfig = tlsConfig + + // Because we create a custom TLSClientConfig, we have to opt-in to HTTP/2. + // See https://github.com/golang/go/issues/14275 + if err := http2.ConfigureTransport(tr); err != nil { + return nil, err + } + } + return tr, nil } From f2820593b0933aa97acf7eba1125d1eebaf87fab Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Mon, 25 Feb 2019 23:07:49 +0000 Subject: [PATCH 005/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 1e77e1200..f47fac1f4 100644 --- a/api.go +++ b/api.go @@ -102,7 +102,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.18" + libraryVersion = "v6.0.19" ) // User Agent should always following the below style. From 59af836a7e6d99cbefa093475fbde0a4552d483f Mon Sep 17 00:00:00 2001 From: poornas Date: Wed, 27 Feb 2019 10:09:23 -0800 Subject: [PATCH 006/215] Filter Expires from user defined metadata (#1076) --- api-datatypes.go | 1 + api-stat.go | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/api-datatypes.go b/api-datatypes.go index 63fc08905..77e987d29 100644 --- a/api-datatypes.go +++ b/api-datatypes.go @@ -41,6 +41,7 @@ type ObjectInfo struct { LastModified time.Time `json:"lastModified"` // Date and time the object was last modified. Size int64 `json:"size"` // Size in bytes of the object. ContentType string `json:"contentType"` // A standard MIME type describing the format of the object data. + Expires time.Time `json:"expires"` // The date and time at which the object is no longer able to be cached. // Collection of additional metadata on the object. // eg: x-amz-meta-*, content-encoding etc. diff --git a/api-stat.go b/api-stat.go index 91e9d3964..f04bac98a 100644 --- a/api-stat.go +++ b/api-stat.go @@ -84,6 +84,7 @@ func extractObjMetadata(header http.Header) http.Header { "Content-Length", "Last-Modified", "Content-Type", + "Expires", }, defaultFilterKeys...) return filterHeader(header, filterKeys) } @@ -170,6 +171,11 @@ func (c Client) statObject(ctx context.Context, bucketName, objectName string, o contentType = "application/octet-stream" } + expiryStr := resp.Header.Get("Expires") + var expTime time.Time + if t, err := time.Parse(http.TimeFormat, expiryStr); err == nil { + expTime = t.UTC() + } // Save object metadata info. return ObjectInfo{ ETag: md5sum, @@ -177,6 +183,7 @@ func (c Client) statObject(ctx context.Context, bucketName, objectName string, o Size: size, LastModified: date, ContentType: contentType, + Expires: expTime, // Extract only the relevant header keys describing the object. // following function filters out a list of standard set of keys // which are not part of object metadata. From f6ad55a0392233db1ddfde0c2c6982288379f844 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Mon, 4 Mar 2019 20:04:59 +0000 Subject: [PATCH 007/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index f47fac1f4..ca18aea7a 100644 --- a/api.go +++ b/api.go @@ -102,7 +102,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.19" + libraryVersion = "v6.0.20" ) // User Agent should always following the below style. From b32976861da0f63fff41eb3fc8b93c313d19ac50 Mon Sep 17 00:00:00 2001 From: poornas Date: Thu, 7 Mar 2019 17:36:36 -0800 Subject: [PATCH 008/215] Omit empty CSV options for s3 select api (#1086) --- api-select.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/api-select.go b/api-select.go index 643abef8d..71c5838b3 100644 --- a/api-select.go +++ b/api-select.go @@ -90,19 +90,19 @@ type ParquetInputOptions struct{} type CSVInputOptions struct { FileHeaderInfo CSVFileHeaderInfo RecordDelimiter string - FieldDelimiter string - QuoteCharacter string - QuoteEscapeCharacter string - Comments string + FieldDelimiter string `xml:",omitempty"` + QuoteCharacter string `xml:",omitempty"` + QuoteEscapeCharacter string `xml:",omitempty"` + Comments string `xml:",omitempty"` } // CSVOutputOptions csv output specific options type CSVOutputOptions struct { - QuoteFields CSVQuoteFields + QuoteFields CSVQuoteFields `xml:",omitempty"` RecordDelimiter string - FieldDelimiter string - QuoteCharacter string - QuoteEscapeCharacter string + FieldDelimiter string `xml:",omitempty"` + QuoteCharacter string `xml:",omitempty"` + QuoteEscapeCharacter string `xml:",omitempty"` } // JSONInputOptions json input specific options From 5df22878d3cdee48fbb086449268b1a465147c7d Mon Sep 17 00:00:00 2001 From: Anis Elleuch Date: Tue, 12 Mar 2019 01:04:18 +0100 Subject: [PATCH 009/215] CopyObjectPart: Add x-amz-copy-source-if-match as condition (#1078) This adds source etag match condition in copy object part, this change is obvious but also tests a use case in Minio server in different copy configuration. --- functional_tests.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/functional_tests.go b/functional_tests.go index a5aff8b69..24f920d08 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -6627,6 +6627,9 @@ func testSSECEncryptedToSSECCopyObjectPart() { for k, v := range header { metadata[k] = v[0] } + + metadata["x-amz-copy-source-if-match"] = objInfo.ETag + // First of three parts fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { @@ -6774,6 +6777,9 @@ func testSSECEncryptedToUnencryptedCopyPart() { for k, v := range header { metadata[k] = v[0] } + + metadata["x-amz-copy-source-if-match"] = objInfo.ETag + // First of three parts fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { @@ -6923,6 +6929,9 @@ func testSSECEncryptedToSSES3CopyObjectPart() { for k, v := range header { metadata[k] = v[0] } + + metadata["x-amz-copy-source-if-match"] = objInfo.ETag + // First of three parts fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { @@ -7069,6 +7078,9 @@ func testUnencryptedToSSECCopyObjectPart() { for k, v := range header { metadata[k] = v[0] } + + metadata["x-amz-copy-source-if-match"] = objInfo.ETag + // First of three parts fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { @@ -7212,6 +7224,9 @@ func testUnencryptedToUnencryptedCopyPart() { for k, v := range header { metadata[k] = v[0] } + + metadata["x-amz-copy-source-if-match"] = objInfo.ETag + // First of three parts fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { @@ -7358,6 +7373,9 @@ func testUnencryptedToSSES3CopyObjectPart() { for k, v := range header { metadata[k] = v[0] } + + metadata["x-amz-copy-source-if-match"] = objInfo.ETag + // First of three parts fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { @@ -7504,6 +7522,9 @@ func testSSES3EncryptedToSSECCopyObjectPart() { for k, v := range header { metadata[k] = v[0] } + + metadata["x-amz-copy-source-if-match"] = objInfo.ETag + // First of three parts fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { @@ -7648,6 +7669,9 @@ func testSSES3EncryptedToUnencryptedCopyPart() { for k, v := range header { metadata[k] = v[0] } + + metadata["x-amz-copy-source-if-match"] = objInfo.ETag + // First of three parts fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { @@ -7795,6 +7819,9 @@ func testSSES3EncryptedToSSES3CopyObjectPart() { for k, v := range header { metadata[k] = v[0] } + + metadata["x-amz-copy-source-if-match"] = objInfo.ETag + // First of three parts fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { From 5d20267d970d9e514bbcb88b37b059bb5321ff60 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Wed, 13 Mar 2019 21:28:32 +0000 Subject: [PATCH 010/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index ca18aea7a..1df7cd052 100644 --- a/api.go +++ b/api.go @@ -102,7 +102,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.20" + libraryVersion = "v6.0.21" ) // User Agent should always following the below style. From a937bac96010803c5460f117490b21f96151c796 Mon Sep 17 00:00:00 2001 From: Anis Elleuch Date: Tue, 26 Mar 2019 18:59:56 +0100 Subject: [PATCH 011/215] tests: Add copy SSEC Multipart to SSEC use case (#1092) A new issue is detected in Minio server when we try to copy a part (CopyObjectPart handler) from a multipart encrypted object. Hence, adding this new use case in functional tests. --- functional_tests.go | 172 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) diff --git a/functional_tests.go b/functional_tests.go index 24f920d08..cfb05e18b 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -6551,6 +6551,177 @@ func testDecryptedCopyObject() { successLogger(testName, function, args, startTime).Info() } +func testSSECMultipartEncryptedToSSECCopyObjectPart() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObjectPart(destination, source)" + args := map[string]interface{}{} + + // Instantiate new minio client object + client, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + return + } + + // Instantiate new core client object. + c := minio.Core{client} + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + } + defer cleanupBucket(bucketName, client) + // Make a buffer with 6MB of data + buf := bytes.Repeat([]byte("abcdef"), 1024*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + password := "correct horse battery staple" + srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) + + // Upload a 6MB object using multipart mechanism + uploadID, err := c.NewMultipartUpload(bucketName, objectName, minio.PutObjectOptions{ServerSideEncryption: srcencryption}) + if err != nil { + logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) + } + + var completeParts []minio.CompletePart + + part, err := c.PutObjectPart(bucketName, objectName, uploadID, 1, bytes.NewReader(buf[:5*1024*1024]), 5*1024*1024, "", "", srcencryption) + if err != nil { + logError(testName, function, args, startTime, "", "PutObjectPart call failed", err) + } + completeParts = append(completeParts, minio.CompletePart{PartNumber: part.PartNumber, ETag: part.ETag}) + + part, err = c.PutObjectPart(bucketName, objectName, uploadID, 2, bytes.NewReader(buf[5*1024*1024:]), 1024*1024, "", "", srcencryption) + if err != nil { + logError(testName, function, args, startTime, "", "PutObjectPart call failed", err) + } + completeParts = append(completeParts, minio.CompletePart{PartNumber: part.PartNumber, ETag: part.ETag}) + + // Complete the multipart upload + _, err = c.CompleteMultipartUpload(bucketName, objectName, uploadID, completeParts) + if err != nil { + logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) + } + + // Stat the object and check its length matches + objInfo, err := c.StatObject(bucketName, objectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: srcencryption}}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject call failed", err) + } + + destBucketName := bucketName + destObjectName := objectName + "-dest" + dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName)) + + uploadID, err = c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) + if err != nil { + logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) + } + + // Content of the destination object will be two copies of + // `objectName` concatenated, followed by first byte of + // `objectName`. + metadata := make(map[string]string) + header := make(http.Header) + encrypt.SSECopy(srcencryption).Marshal(header) + dstencryption.Marshal(header) + for k, v := range header { + metadata[k] = v[0] + } + + metadata["x-amz-copy-source-if-match"] = objInfo.ETag + + // First of three parts + fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Second of three parts + sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Last of three parts + lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Complete the multipart upload + _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + if err != nil { + logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) + } + + // Stat the object and check its length matches + objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject call failed", err) + } + + if objInfo.Size != (6*1024*1024)*2+1 { + logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) + } + + // Now we read the data back + getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} + getOpts.SetRange(0, 6*1024*1024-1) + r, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject call failed", err) + } + getBuf := make([]byte, 6*1024*1024) + _, err = io.ReadFull(r, getBuf) + if err != nil { + logError(testName, function, args, startTime, "", "Read buffer failed", err) + } + if !bytes.Equal(getBuf, buf) { + logError(testName, function, args, startTime, "", "Got unexpected data in first 6MB", err) + } + + getOpts.SetRange(6*1024*1024, 0) + r, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject call failed", err) + } + getBuf = make([]byte, 6*1024*1024+1) + _, err = io.ReadFull(r, getBuf) + if err != nil { + logError(testName, function, args, startTime, "", "Read buffer failed", err) + } + if !bytes.Equal(getBuf[:6*1024*1024], buf) { + logError(testName, function, args, startTime, "", "Got unexpected data in second 6MB", err) + } + if getBuf[6*1024*1024] != buf[0] { + logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) + } + + successLogger(testName, function, args, startTime).Info() + + // Do not need to remove destBucketName its same as bucketName. +} + // Test Core CopyObjectPart implementation func testSSECEncryptedToSSECCopyObjectPart() { // initialize logging params @@ -9869,6 +10040,7 @@ func main() { testEncryptedEmptyObject() testDecryptedCopyObject() testSSECEncryptedToSSECCopyObjectPart() + testSSECMultipartEncryptedToSSECCopyObjectPart() testSSECEncryptedToUnencryptedCopyPart() testUnencryptedToSSECCopyObjectPart() testUnencryptedToUnencryptedCopyPart() From 5325257a208fc630aaaac31bc00789acbc998c14 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Wed, 27 Mar 2019 20:36:52 +0000 Subject: [PATCH 012/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 1df7cd052..40a5d881b 100644 --- a/api.go +++ b/api.go @@ -102,7 +102,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.21" + libraryVersion = "v6.0.22" ) // User Agent should always following the below style. From 30a777d088e69c23131eabfdf9b6aabf88a4d1df Mon Sep 17 00:00:00 2001 From: Kanagaraj M Date: Wed, 3 Apr 2019 13:00:16 +0530 Subject: [PATCH 013/215] remove go-music-player-app links from docs (#1094) --- README.md | 1 - README_zh_CN.md | 3 +-- docs/API.md | 5 ----- docs/zh_CN/API.md | 5 ----- 4 files changed, 1 insertion(+), 13 deletions(-) diff --git a/README.md b/README.md index ad9d5e60b..36647fd56 100644 --- a/README.md +++ b/README.md @@ -227,7 +227,6 @@ The full API Reference is available here. ## Explore Further * [Complete Documentation](https://docs.minio.io) * [Minio Go Client SDK API Reference](https://docs.minio.io/docs/golang-client-api-reference) -* [Go Music Player App Full Application Example](https://docs.minio.io/docs/go-music-player-app) ## Contribute [Contributors Guide](https://github.com/minio/minio-go/blob/master/CONTRIBUTING.md) diff --git a/README_zh_CN.md b/README_zh_CN.md index a5acf199e..62fecb63a 100644 --- a/README_zh_CN.md +++ b/README_zh_CN.md @@ -234,8 +234,7 @@ mc ls play/mymusic/ ## 了解更多 * [完整文档](https://docs.minio.io) -* [Minio Go Client SDK API文档](https://docs.minio.io/docs/golang-client-api-reference) -* [Go 音乐播放器完整示例](https://docs.minio.io/docs/go-music-player-app) +* [Minio Go Client SDK API文档](https://docs.minio.io/docs/golang-client-api-reference) ## 贡献 [贡献指南](https://github.com/minio/minio-go/blob/master/docs/zh_CN/CONTRIBUTING.md) diff --git a/docs/API.md b/docs/API.md index 577821673..a4c8d7d7b 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1623,8 +1623,3 @@ __Parameters__ | Param | Type | Description | |---|---|---| |`acceleratedEndpoint` | _string_ | Set to new S3 transfer acceleration endpoint.| - - -## 8. Explore Further - -- [Build your own Go Music Player App example](https://docs.minio.io/docs/go-music-player-app) diff --git a/docs/zh_CN/API.md b/docs/zh_CN/API.md index ca80586a4..dacdf4b44 100644 --- a/docs/zh_CN/API.md +++ b/docs/zh_CN/API.md @@ -1779,8 +1779,3 @@ __参数__ | 参数 | 类型 | 描述 | |---|---|---| |`acceleratedEndpoint` | _string_ | 设置新的S3传输加速endpoint。| - - -## 8. 了解更多 - -- [用Go语言创建属于你的音乐播放器APP示例](https://docs.minio.io/docs/go-music-player-app) From 93e12e097cf3aec86c28ceeb7a960a29b2302a3a Mon Sep 17 00:00:00 2001 From: kannappanr <30541348+kannappanr@users.noreply.github.com> Date: Sun, 7 Apr 2019 15:54:46 -0700 Subject: [PATCH 014/215] Replace Minio with MinIO and minio.io with min.io (#1096) --- MAINTAINERS.md | 6 +- Makefile | 4 +- NOTICE | 2 +- README.md | 98 +++--- README_zh_CN.md | 102 +++--- api-compose-object.go | 4 +- api-compose-object_test.go | 4 +- api-datatypes.go | 4 +- api-error-response.go | 4 +- api-error-response_test.go | 4 +- api-get-lifecycle.go | 4 +- api-get-object-acl.go | 4 +- api-get-object-context.go | 4 +- api-get-object-file.go | 4 +- api-get-object.go | 4 +- api-get-options.go | 4 +- api-get-policy.go | 4 +- api-list.go | 4 +- api-notification.go | 4 +- api-presigned.go | 4 +- api-put-bucket.go | 4 +- api-put-object-common.go | 4 +- api-put-object-context.go | 4 +- api-put-object-copy.go | 4 +- api-put-object-file-context.go | 4 +- api-put-object-file.go | 4 +- api-put-object-multipart.go | 4 +- api-put-object-streaming.go | 4 +- api-put-object.go | 4 +- api-put-object_test.go | 4 +- api-remove.go | 4 +- api-s3-datatypes.go | 4 +- api-select.go | 4 +- api-stat.go | 4 +- api.go | 8 +- api_unit_test.go | 6 +- bucket-cache.go | 4 +- bucket-cache_test.go | 10 +- bucket-notification.go | 4 +- constants.go | 4 +- core.go | 4 +- core_test.go | 16 +- docs/API.md | 12 +- docs/checker.go.template | 2 +- docs/validator.go | 6 +- docs/zh_CN/API.md | 10 +- examples/minio/listenbucketnotification.go | 6 +- examples/s3/bucketexists.go | 4 +- examples/s3/composeobject.go | 4 +- examples/s3/copyobject.go | 4 +- examples/s3/fgetobject-context.go | 4 +- examples/s3/fgetobject.go | 4 +- examples/s3/fputencrypted-object.go | 4 +- examples/s3/fputobject-context.go | 4 +- examples/s3/fputobject.go | 4 +- examples/s3/get-encrypted-object.go | 4 +- examples/s3/getbucketlifecycle.go | 4 +- examples/s3/getbucketnotification.go | 4 +- examples/s3/getbucketpolicy.go | 4 +- examples/s3/getobject-client-encryption.go | 4 +- examples/s3/getobject-context.go | 4 +- examples/s3/getobject.go | 4 +- examples/s3/getobjectacl.go | 4 +- examples/s3/listbuckets.go | 4 +- examples/s3/listincompleteuploads.go | 4 +- examples/s3/listobjects-N.go | 4 +- examples/s3/listobjects.go | 4 +- examples/s3/listobjectsV2.go | 4 +- examples/s3/makebucket.go | 4 +- examples/s3/presignedgetobject.go | 4 +- examples/s3/presignedheadobject.go | 4 +- examples/s3/presignedpostpolicy.go | 4 +- examples/s3/presignedputobject.go | 4 +- examples/s3/put-encrypted-object.go | 4 +- examples/s3/putobject-client-encryption.go | 4 +- examples/s3/putobject-context.go | 4 +- examples/s3/putobject-getobject-sse.go | 4 +- examples/s3/putobject-progress.go | 4 +- examples/s3/putobject-s3-accelerate.go | 4 +- examples/s3/putobject-streaming.go | 4 +- examples/s3/putobject.go | 4 +- examples/s3/removeallbucketnotification.go | 4 +- examples/s3/removebucket.go | 4 +- examples/s3/removeincompleteupload.go | 4 +- examples/s3/removeobject.go | 4 +- examples/s3/removeobjects.go | 4 +- examples/s3/selectobject.go | 4 +- examples/s3/setbucketlifecycle.go | 4 +- examples/s3/setbucketnotification.go | 4 +- examples/s3/setbucketpolicy.go | 4 +- examples/s3/statobject.go | 4 +- functional_tests.go | 290 +++++++++--------- get-options_test.go | 4 +- hook-reader.go | 4 +- pkg/credentials/chain.go | 4 +- pkg/credentials/chain_test.go | 4 +- pkg/credentials/config.json.sample | 2 +- pkg/credentials/credentials.go | 4 +- pkg/credentials/credentials_test.go | 4 +- pkg/credentials/doc.go | 4 +- pkg/credentials/env_aws.go | 4 +- pkg/credentials/env_minio.go | 4 +- pkg/credentials/env_test.go | 4 +- pkg/credentials/file_aws_credentials.go | 4 +- pkg/credentials/file_minio_client.go | 6 +- pkg/credentials/file_test.go | 4 +- pkg/credentials/iam_aws.go | 4 +- pkg/credentials/iam_aws_test.go | 4 +- pkg/credentials/signature-type.go | 4 +- pkg/credentials/static.go | 4 +- pkg/credentials/static_test.go | 4 +- pkg/credentials/sts_client_grants.go | 14 +- pkg/credentials/sts_web_identity.go | 14 +- pkg/encrypt/server-side.go | 4 +- pkg/policy/bucket-policy-condition.go | 4 +- pkg/policy/bucket-policy-condition_test.go | 4 +- pkg/policy/bucket-policy.go | 4 +- pkg/policy/bucket-policy_test.go | 4 +- pkg/s3signer/request-signature-streaming.go | 4 +- .../request-signature-streaming_test.go | 4 +- pkg/s3signer/request-signature-v2.go | 4 +- pkg/s3signer/request-signature-v2_test.go | 4 +- pkg/s3signer/request-signature-v4.go | 4 +- pkg/s3signer/request-signature-v4_test.go | 4 +- pkg/s3signer/request-signature_test.go | 4 +- pkg/s3signer/test-utils_test.go | 4 +- pkg/s3signer/utils.go | 4 +- pkg/s3signer/utils_test.go | 4 +- pkg/s3utils/utils.go | 4 +- pkg/s3utils/utils_test.go | 4 +- pkg/set/stringset.go | 4 +- pkg/set/stringset_test.go | 4 +- post-policy.go | 4 +- retry-continous.go | 4 +- retry.go | 4 +- s3-endpoints.go | 4 +- s3-error.go | 4 +- test-utils_test.go | 4 +- transport.go | 4 +- utils.go | 4 +- utils_test.go | 4 +- 141 files changed, 551 insertions(+), 551 deletions(-) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 17973078e..f640dfb9f 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -5,7 +5,7 @@ Please go through this link [Maintainer Responsibility](https://gist.github.com/abperiasamy/f4d9b31d3186bbd26522) ### Making new releases -Tag and sign your release commit, additionally this step requires you to have access to Minio's trusted private key. +Tag and sign your release commit, additionally this step requires you to have access to MinIO's trusted private key. ```sh $ export GNUPGHOME=/media/${USER}/minio/trusted $ git tag -s 4.0.0 @@ -23,11 +23,11 @@ $ grep libraryVersion api.go Commit your changes ``` -$ git commit -a -m "Update version for next release" --author "Minio Trusted " +$ git commit -a -m "Update version for next release" --author "MinIO Trusted " ``` ### Announce -Announce new release by adding release notes at https://github.com/minio/minio-go/releases from `trusted@minio.io` account. Release notes requires two sections `highlights` and `changelog`. Highlights is a bulleted list of salient features in this release and Changelog contains list of all commits since the last release. +Announce new release by adding release notes at https://github.com/minio/minio-go/releases from `trusted@min.io` account. Release notes requires two sections `highlights` and `changelog`. Highlights is a bulleted list of salient features in this release and Changelog contains list of all commits since the last release. To generate `changelog` ```sh diff --git a/Makefile b/Makefile index bad81ffaf..51c8ca266 100644 --- a/Makefile +++ b/Makefile @@ -3,10 +3,10 @@ all: checks checks: @go get -t ./... @go vet ./... - @SERVER_ENDPOINT=play.minio.io:9000 ACCESS_KEY=Q3AM3UQ867SPQQA43P2F SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG ENABLE_HTTPS=1 MINT_MODE=full go test -race -v ./... + @SERVER_ENDPOINT=play.min.io:9000 ACCESS_KEY=Q3AM3UQ867SPQQA43P2F SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG ENABLE_HTTPS=1 MINT_MODE=full go test -race -v ./... @go get github.com/dustin/go-humanize/... @go get github.com/sirupsen/logrus/... - @SERVER_ENDPOINT=play.minio.io:9000 ACCESS_KEY=Q3AM3UQ867SPQQA43P2F SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG ENABLE_HTTPS=1 MINT_MODE=full go run functional_tests.go + @SERVER_ENDPOINT=play.min.io:9000 ACCESS_KEY=Q3AM3UQ867SPQQA43P2F SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG ENABLE_HTTPS=1 MINT_MODE=full go run functional_tests.go @mkdir -p /tmp/examples && for i in $(echo examples/s3/*); do go build -o /tmp/examples/$(basename ${i:0:-3}) ${i}; done @go get -u github.com/a8m/mark/... @go get -u github.com/minio/cli/... diff --git a/NOTICE b/NOTICE index c521791c5..fc6c8808e 100644 --- a/NOTICE +++ b/NOTICE @@ -1,2 +1,2 @@ minio-go -Copyright 2015-2017 Minio, Inc. \ No newline at end of file +Copyright 2015-2017 MinIO, Inc. \ No newline at end of file diff --git a/README.md b/README.md index 36647fd56..5464390fa 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,18 @@ -# Minio Go Client SDK for Amazon S3 Compatible Cloud Storage [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Sourcegraph](https://sourcegraph.com/github.com/minio/minio-go/-/badge.svg)](https://sourcegraph.com/github.com/minio/minio-go?badge) [![Apache V2 License](http://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/minio/minio-go/blob/master/LICENSE) +# MinIO Go Client SDK for Amazon S3 Compatible Cloud Storage [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Sourcegraph](https://sourcegraph.com/github.com/minio/minio-go/-/badge.svg)](https://sourcegraph.com/github.com/minio/minio-go?badge) [![Apache V2 License](http://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/minio/minio-go/blob/master/LICENSE) -The Minio Go Client SDK provides simple APIs to access any Amazon S3 compatible object storage. +The MinIO Go Client SDK provides simple APIs to access any Amazon S3 compatible object storage. -This quickstart guide will show you how to install the Minio client SDK, connect to Minio, and provide a walkthrough for a simple file uploader. For a complete list of APIs and examples, please take a look at the [Go Client API Reference](https://docs.minio.io/docs/golang-client-api-reference). +This quickstart guide will show you how to install the MinIO client SDK, connect to MinIO, and provide a walkthrough for a simple file uploader. For a complete list of APIs and examples, please take a look at the [Go Client API Reference](https://docs.min.io/docs/golang-client-api-reference). -This document assumes that you have a working [Go development environment](https://docs.minio.io/docs/how-to-install-golang). +This document assumes that you have a working [Go development environment](https://docs.min.io/docs/how-to-install-golang). ## Download from Github ```sh go get -u github.com/minio/minio-go ``` -## Initialize Minio Client -Minio client requires the following four parameters specified to connect to an Amazon S3 compatible object storage. +## Initialize MinIO Client +MinIO client requires the following four parameters specified to connect to an Amazon S3 compatible object storage. | Parameter | Description| | :--- | :--- | @@ -31,7 +31,7 @@ import ( ) func main() { - endpoint := "play.minio.io:9000" + endpoint := "play.min.io:9000" accessKeyID := "Q3AM3UQ867SPQQA43P2F" secretAccessKey := "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" useSSL := true @@ -49,7 +49,7 @@ func main() { ## Quick Start Example - File Uploader This example program connects to an object storage server, creates a bucket and uploads a file to the bucket. -We will use the Minio server running at [https://play.minio.io:9000](https://play.minio.io:9000) in this example. Feel free to use this service for testing and development. Access credentials shown in this example are open to the public. +We will use the MinIO server running at [https://play.min.io:9000](https://play.min.io:9000) in this example. Feel free to use this service for testing and development. Access credentials shown in this example are open to the public. ### FileUploader.go ```go @@ -61,7 +61,7 @@ import ( ) func main() { - endpoint := "play.minio.io:9000" + endpoint := "play.min.io:9000" accessKeyID := "Q3AM3UQ867SPQQA43P2F" secretAccessKey := "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" useSSL := true @@ -117,58 +117,58 @@ mc ls play/mymusic/ ## API Reference The full API Reference is available here. -* [Complete API Reference](https://docs.minio.io/docs/golang-client-api-reference) +* [Complete API Reference](https://docs.min.io/docs/golang-client-api-reference) ### API Reference : Bucket Operations -* [`MakeBucket`](https://docs.minio.io/docs/golang-client-api-reference#MakeBucket) -* [`ListBuckets`](https://docs.minio.io/docs/golang-client-api-reference#ListBuckets) -* [`BucketExists`](https://docs.minio.io/docs/golang-client-api-reference#BucketExists) -* [`RemoveBucket`](https://docs.minio.io/docs/golang-client-api-reference#RemoveBucket) -* [`ListObjects`](https://docs.minio.io/docs/golang-client-api-reference#ListObjects) -* [`ListObjectsV2`](https://docs.minio.io/docs/golang-client-api-reference#ListObjectsV2) -* [`ListIncompleteUploads`](https://docs.minio.io/docs/golang-client-api-reference#ListIncompleteUploads) +* [`MakeBucket`](https://docs.min.io/docs/golang-client-api-reference#MakeBucket) +* [`ListBuckets`](https://docs.min.io/docs/golang-client-api-reference#ListBuckets) +* [`BucketExists`](https://docs.min.io/docs/golang-client-api-reference#BucketExists) +* [`RemoveBucket`](https://docs.min.io/docs/golang-client-api-reference#RemoveBucket) +* [`ListObjects`](https://docs.min.io/docs/golang-client-api-reference#ListObjects) +* [`ListObjectsV2`](https://docs.min.io/docs/golang-client-api-reference#ListObjectsV2) +* [`ListIncompleteUploads`](https://docs.min.io/docs/golang-client-api-reference#ListIncompleteUploads) ### API Reference : Bucket policy Operations -* [`SetBucketPolicy`](https://docs.minio.io/docs/golang-client-api-reference#SetBucketPolicy) -* [`GetBucketPolicy`](https://docs.minio.io/docs/golang-client-api-reference#GetBucketPolicy) +* [`SetBucketPolicy`](https://docs.min.io/docs/golang-client-api-reference#SetBucketPolicy) +* [`GetBucketPolicy`](https://docs.min.io/docs/golang-client-api-reference#GetBucketPolicy) ### API Reference : Bucket notification Operations -* [`SetBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#SetBucketNotification) -* [`GetBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#GetBucketNotification) -* [`RemoveAllBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#RemoveAllBucketNotification) -* [`ListenBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#ListenBucketNotification) (Minio Extension) +* [`SetBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#SetBucketNotification) +* [`GetBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#GetBucketNotification) +* [`RemoveAllBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#RemoveAllBucketNotification) +* [`ListenBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#ListenBucketNotification) (MinIO Extension) ### API Reference : File Object Operations -* [`FPutObject`](https://docs.minio.io/docs/golang-client-api-reference#FPutObject) -* [`FGetObject`](https://docs.minio.io/docs/golang-client-api-reference#FGetObject) -* [`FPutObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#FPutObjectWithContext) -* [`FGetObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#FGetObjectWithContext) +* [`FPutObject`](https://docs.min.io/docs/golang-client-api-reference#FPutObject) +* [`FGetObject`](https://docs.min.io/docs/golang-client-api-reference#FGetObject) +* [`FPutObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#FPutObjectWithContext) +* [`FGetObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#FGetObjectWithContext) ### API Reference : Object Operations -* [`GetObject`](https://docs.minio.io/docs/golang-client-api-reference#GetObject) -* [`PutObject`](https://docs.minio.io/docs/golang-client-api-reference#PutObject) -* [`GetObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#GetObjectWithContext) -* [`PutObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#PutObjectWithContext) -* [`PutObjectStreaming`](https://docs.minio.io/docs/golang-client-api-reference#PutObjectStreaming) -* [`StatObject`](https://docs.minio.io/docs/golang-client-api-reference#StatObject) -* [`CopyObject`](https://docs.minio.io/docs/golang-client-api-reference#CopyObject) -* [`RemoveObject`](https://docs.minio.io/docs/golang-client-api-reference#RemoveObject) -* [`RemoveObjects`](https://docs.minio.io/docs/golang-client-api-reference#RemoveObjects) -* [`RemoveIncompleteUpload`](https://docs.minio.io/docs/golang-client-api-reference#RemoveIncompleteUpload) -* [`SelectObjectContent`](https://docs.minio.io/docs/golang-client-api-reference#SelectObjectContent) +* [`GetObject`](https://docs.min.io/docs/golang-client-api-reference#GetObject) +* [`PutObject`](https://docs.min.io/docs/golang-client-api-reference#PutObject) +* [`GetObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#GetObjectWithContext) +* [`PutObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#PutObjectWithContext) +* [`PutObjectStreaming`](https://docs.min.io/docs/golang-client-api-reference#PutObjectStreaming) +* [`StatObject`](https://docs.min.io/docs/golang-client-api-reference#StatObject) +* [`CopyObject`](https://docs.min.io/docs/golang-client-api-reference#CopyObject) +* [`RemoveObject`](https://docs.min.io/docs/golang-client-api-reference#RemoveObject) +* [`RemoveObjects`](https://docs.min.io/docs/golang-client-api-reference#RemoveObjects) +* [`RemoveIncompleteUpload`](https://docs.min.io/docs/golang-client-api-reference#RemoveIncompleteUpload) +* [`SelectObjectContent`](https://docs.min.io/docs/golang-client-api-reference#SelectObjectContent) ### API Reference : Presigned Operations -* [`PresignedGetObject`](https://docs.minio.io/docs/golang-client-api-reference#PresignedGetObject) -* [`PresignedPutObject`](https://docs.minio.io/docs/golang-client-api-reference#PresignedPutObject) -* [`PresignedHeadObject`](https://docs.minio.io/docs/golang-client-api-reference#PresignedHeadObject) -* [`PresignedPostPolicy`](https://docs.minio.io/docs/golang-client-api-reference#PresignedPostPolicy) +* [`PresignedGetObject`](https://docs.min.io/docs/golang-client-api-reference#PresignedGetObject) +* [`PresignedPutObject`](https://docs.min.io/docs/golang-client-api-reference#PresignedPutObject) +* [`PresignedHeadObject`](https://docs.min.io/docs/golang-client-api-reference#PresignedHeadObject) +* [`PresignedPostPolicy`](https://docs.min.io/docs/golang-client-api-reference#PresignedPostPolicy) ### API Reference : Client custom settings -* [`SetAppInfo`](http://docs.minio.io/docs/golang-client-api-reference#SetAppInfo) -* [`SetCustomTransport`](http://docs.minio.io/docs/golang-client-api-reference#SetCustomTransport) -* [`TraceOn`](http://docs.minio.io/docs/golang-client-api-reference#TraceOn) -* [`TraceOff`](http://docs.minio.io/docs/golang-client-api-reference#TraceOff) +* [`SetAppInfo`](http://docs.min.io/docs/golang-client-api-reference#SetAppInfo) +* [`SetCustomTransport`](http://docs.min.io/docs/golang-client-api-reference#SetCustomTransport) +* [`TraceOn`](http://docs.min.io/docs/golang-client-api-reference#TraceOn) +* [`TraceOff`](http://docs.min.io/docs/golang-client-api-reference#TraceOff) ## Full Examples @@ -194,7 +194,7 @@ The full API Reference is available here. * [setbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketnotification.go) * [getbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketnotification.go) * [removeallbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeallbucketnotification.go) -* [listenbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/minio/listenbucketnotification.go) (Minio Extension) +* [listenbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/minio/listenbucketnotification.go) (MinIO Extension) ### Full Examples : File Object Operations * [fputobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/fputobject.go) @@ -225,8 +225,8 @@ The full API Reference is available here. * [presignedpostpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedpostpolicy.go) ## Explore Further -* [Complete Documentation](https://docs.minio.io) -* [Minio Go Client SDK API Reference](https://docs.minio.io/docs/golang-client-api-reference) +* [Complete Documentation](https://docs.min.io) +* [MinIO Go Client SDK API Reference](https://docs.min.io/docs/golang-client-api-reference) ## Contribute [Contributors Guide](https://github.com/minio/minio-go/blob/master/CONTRIBUTING.md) diff --git a/README_zh_CN.md b/README_zh_CN.md index 62fecb63a..061f41743 100644 --- a/README_zh_CN.md +++ b/README_zh_CN.md @@ -1,12 +1,12 @@ -# 适用于与Amazon S3兼容云存储的Minio Go SDK [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Sourcegraph](https://sourcegraph.com/github.com/minio/minio-go/-/badge.svg)](https://sourcegraph.com/github.com/minio/minio-go?badge) +# 适用于与Amazon S3兼容云存储的MinIO Go SDK [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Sourcegraph](https://sourcegraph.com/github.com/minio/minio-go/-/badge.svg)](https://sourcegraph.com/github.com/minio/minio-go?badge) -Minio Go Client SDK提供了简单的API来访问任何与Amazon S3兼容的对象存储服务。 +MinIO Go Client SDK提供了简单的API来访问任何与Amazon S3兼容的对象存储服务。 **支持的云存储:** - AWS Signature Version 4 - Amazon S3 - - Minio + - MinIO - AWS Signature Version 2 - Google Cloud Storage (兼容模式) @@ -14,17 +14,17 @@ Minio Go Client SDK提供了简单的API来访问任何与Amazon S3兼容的对 - Ceph Object Gateway - Riak CS -本文我们将学习如何安装Minio client SDK,连接到Minio,并提供一下文件上传的示例。对于完整的API以及示例,请参考[Go Client API Reference](https://docs.minio.io/docs/golang-client-api-reference)。 +本文我们将学习如何安装MinIO client SDK,连接到MinIO,并提供一下文件上传的示例。对于完整的API以及示例,请参考[Go Client API Reference](https://docs.min.io/docs/golang-client-api-reference)。 -本文假设你已经有 [Go开发环境](https://docs.minio.io/docs/how-to-install-golang)。 +本文假设你已经有 [Go开发环境](https://docs.min.io/docs/how-to-install-golang)。 ## 从Github下载 ```sh go get -u github.com/minio/minio-go ``` -## 初始化Minio Client -Minio client需要以下4个参数来连接与Amazon S3兼容的对象存储。 +## 初始化MinIO Client +MinIO client需要以下4个参数来连接与Amazon S3兼容的对象存储。 | 参数 | 描述| | :--- | :--- | @@ -43,7 +43,7 @@ import ( ) func main() { - endpoint := "play.minio.io:9000" + endpoint := "play.min.io:9000" accessKeyID := "Q3AM3UQ867SPQQA43P2F" secretAccessKey := "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" useSSL := true @@ -61,7 +61,7 @@ func main() { ## 示例-文件上传 本示例连接到一个对象存储服务,创建一个存储桶并上传一个文件到存储桶中。 -我们在本示例中使用运行在 [https://play.minio.io:9000](https://play.minio.io:9000) 上的Minio服务,你可以用这个服务来开发和测试。示例中的访问凭据是公开的。 +我们在本示例中使用运行在 [https://play.min.io:9000](https://play.min.io:9000) 上的MinIO服务,你可以用这个服务来开发和测试。示例中的访问凭据是公开的。 ### FileUploader.go ```go @@ -73,7 +73,7 @@ import ( ) func main() { - endpoint := "play.minio.io:9000" + endpoint := "play.min.io:9000" accessKeyID := "Q3AM3UQ867SPQQA43P2F" secretAccessKey := "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" useSSL := true @@ -127,60 +127,60 @@ mc ls play/mymusic/ ## API文档 完整的API文档在这里。 -* [完整API文档](https://docs.minio.io/docs/golang-client-api-reference) +* [完整API文档](https://docs.min.io/docs/golang-client-api-reference) ### API文档 : 操作存储桶 -* [`MakeBucket`](https://docs.minio.io/docs/golang-client-api-reference#MakeBucket) -* [`ListBuckets`](https://docs.minio.io/docs/golang-client-api-reference#ListBuckets) -* [`BucketExists`](https://docs.minio.io/docs/golang-client-api-reference#BucketExists) -* [`RemoveBucket`](https://docs.minio.io/docs/golang-client-api-reference#RemoveBucket) -* [`ListObjects`](https://docs.minio.io/docs/golang-client-api-reference#ListObjects) -* [`ListObjectsV2`](https://docs.minio.io/docs/golang-client-api-reference#ListObjectsV2) -* [`ListIncompleteUploads`](https://docs.minio.io/docs/golang-client-api-reference#ListIncompleteUploads) +* [`MakeBucket`](https://docs.min.io/docs/golang-client-api-reference#MakeBucket) +* [`ListBuckets`](https://docs.min.io/docs/golang-client-api-reference#ListBuckets) +* [`BucketExists`](https://docs.min.io/docs/golang-client-api-reference#BucketExists) +* [`RemoveBucket`](https://docs.min.io/docs/golang-client-api-reference#RemoveBucket) +* [`ListObjects`](https://docs.min.io/docs/golang-client-api-reference#ListObjects) +* [`ListObjectsV2`](https://docs.min.io/docs/golang-client-api-reference#ListObjectsV2) +* [`ListIncompleteUploads`](https://docs.min.io/docs/golang-client-api-reference#ListIncompleteUploads) ### API文档 : 存储桶策略 -* [`SetBucketPolicy`](https://docs.minio.io/docs/golang-client-api-reference#SetBucketPolicy) -* [`GetBucketPolicy`](https://docs.minio.io/docs/golang-client-api-reference#GetBucketPolicy) +* [`SetBucketPolicy`](https://docs.min.io/docs/golang-client-api-reference#SetBucketPolicy) +* [`GetBucketPolicy`](https://docs.min.io/docs/golang-client-api-reference#GetBucketPolicy) ### API文档 : 存储桶通知 -* [`SetBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#SetBucketNotification) -* [`GetBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#GetBucketNotification) -* [`RemoveAllBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#RemoveAllBucketNotification) -* [`ListenBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#ListenBucketNotification) (Minio Extension) +* [`SetBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#SetBucketNotification) +* [`GetBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#GetBucketNotification) +* [`RemoveAllBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#RemoveAllBucketNotification) +* [`ListenBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#ListenBucketNotification) (MinIO Extension) ### API文档 : 操作文件对象 -* [`FPutObject`](https://docs.minio.io/docs/golang-client-api-reference#FPutObject) -* [`FGetObject`](https://docs.minio.io/docs/golang-client-api-reference#FPutObject) -* [`FPutObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#FPutObjectWithContext) -* [`FGetObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#FGetObjectWithContext) +* [`FPutObject`](https://docs.min.io/docs/golang-client-api-reference#FPutObject) +* [`FGetObject`](https://docs.min.io/docs/golang-client-api-reference#FPutObject) +* [`FPutObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#FPutObjectWithContext) +* [`FGetObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#FGetObjectWithContext) ### API文档 : 操作对象 -* [`GetObject`](https://docs.minio.io/docs/golang-client-api-reference#GetObject) -* [`PutObject`](https://docs.minio.io/docs/golang-client-api-reference#PutObject) -* [`GetObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#GetObjectWithContext) -* [`PutObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#PutObjectWithContext) -* [`PutObjectStreaming`](https://docs.minio.io/docs/golang-client-api-reference#PutObjectStreaming) -* [`StatObject`](https://docs.minio.io/docs/golang-client-api-reference#StatObject) -* [`CopyObject`](https://docs.minio.io/docs/golang-client-api-reference#CopyObject) -* [`RemoveObject`](https://docs.minio.io/docs/golang-client-api-reference#RemoveObject) -* [`RemoveObjects`](https://docs.minio.io/docs/golang-client-api-reference#RemoveObjects) -* [`RemoveIncompleteUpload`](https://docs.minio.io/docs/golang-client-api-reference#RemoveIncompleteUpload) +* [`GetObject`](https://docs.min.io/docs/golang-client-api-reference#GetObject) +* [`PutObject`](https://docs.min.io/docs/golang-client-api-reference#PutObject) +* [`GetObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#GetObjectWithContext) +* [`PutObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#PutObjectWithContext) +* [`PutObjectStreaming`](https://docs.min.io/docs/golang-client-api-reference#PutObjectStreaming) +* [`StatObject`](https://docs.min.io/docs/golang-client-api-reference#StatObject) +* [`CopyObject`](https://docs.min.io/docs/golang-client-api-reference#CopyObject) +* [`RemoveObject`](https://docs.min.io/docs/golang-client-api-reference#RemoveObject) +* [`RemoveObjects`](https://docs.min.io/docs/golang-client-api-reference#RemoveObjects) +* [`RemoveIncompleteUpload`](https://docs.min.io/docs/golang-client-api-reference#RemoveIncompleteUpload) ### API文档: 操作加密对象 -* [`GetEncryptedObject`](https://docs.minio.io/docs/golang-client-api-reference#GetEncryptedObject) -* [`PutEncryptedObject`](https://docs.minio.io/docs/golang-client-api-reference#PutEncryptedObject) +* [`GetEncryptedObject`](https://docs.min.io/docs/golang-client-api-reference#GetEncryptedObject) +* [`PutEncryptedObject`](https://docs.min.io/docs/golang-client-api-reference#PutEncryptedObject) ### API文档 : Presigned操作 -* [`PresignedGetObject`](https://docs.minio.io/docs/golang-client-api-reference#PresignedGetObject) -* [`PresignedPutObject`](https://docs.minio.io/docs/golang-client-api-reference#PresignedPutObject) -* [`PresignedHeadObject`](https://docs.minio.io/docs/golang-client-api-reference#PresignedHeadObject) -* [`PresignedPostPolicy`](https://docs.minio.io/docs/golang-client-api-reference#PresignedPostPolicy) +* [`PresignedGetObject`](https://docs.min.io/docs/golang-client-api-reference#PresignedGetObject) +* [`PresignedPutObject`](https://docs.min.io/docs/golang-client-api-reference#PresignedPutObject) +* [`PresignedHeadObject`](https://docs.min.io/docs/golang-client-api-reference#PresignedHeadObject) +* [`PresignedPostPolicy`](https://docs.min.io/docs/golang-client-api-reference#PresignedPostPolicy) ### API文档 : 客户端自定义设置 -* [`SetAppInfo`](http://docs.minio.io/docs/golang-client-api-reference#SetAppInfo) -* [`SetCustomTransport`](http://docs.minio.io/docs/golang-client-api-reference#SetCustomTransport) -* [`TraceOn`](http://docs.minio.io/docs/golang-client-api-reference#TraceOn) -* [`TraceOff`](http://docs.minio.io/docs/golang-client-api-reference#TraceOff) +* [`SetAppInfo`](http://docs.min.io/docs/golang-client-api-reference#SetAppInfo) +* [`SetCustomTransport`](http://docs.min.io/docs/golang-client-api-reference#SetCustomTransport) +* [`TraceOn`](http://docs.min.io/docs/golang-client-api-reference#TraceOn) +* [`TraceOff`](http://docs.min.io/docs/golang-client-api-reference#TraceOff) ## 完整示例 @@ -202,7 +202,7 @@ mc ls play/mymusic/ * [setbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketnotification.go) * [getbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketnotification.go) * [removeallbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeallbucketnotification.go) -* [listenbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/minio/listenbucketnotification.go) (Minio扩展) +* [listenbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/minio/listenbucketnotification.go) (MinIO扩展) ### 完整示例 : 操作文件对象 * [fputobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/fputobject.go) @@ -233,8 +233,8 @@ mc ls play/mymusic/ * [presignedpostpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedpostpolicy.go) ## 了解更多 -* [完整文档](https://docs.minio.io) -* [Minio Go Client SDK API文档](https://docs.minio.io/docs/golang-client-api-reference) +* [完整文档](https://docs.min.io) +* [MinIO Go Client SDK API文档](https://docs.min.io/docs/golang-client-api-reference) ## 贡献 [贡献指南](https://github.com/minio/minio-go/blob/master/docs/zh_CN/CONTRIBUTING.md) diff --git a/api-compose-object.go b/api-compose-object.go index 3ac36c502..b4dddacb2 100644 --- a/api-compose-object.go +++ b/api-compose-object.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017, 2018 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017, 2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-compose-object_test.go b/api-compose-object_test.go index 295bbc263..e9f04d3a6 100644 --- a/api-compose-object_test.go +++ b/api-compose-object_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-datatypes.go b/api-datatypes.go index 77e987d29..3fb47fc10 100644 --- a/api-datatypes.go +++ b/api-datatypes.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-error-response.go b/api-error-response.go index 0170b8de8..5dc8a0a90 100644 --- a/api-error-response.go +++ b/api-error-response.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-error-response_test.go b/api-error-response_test.go index 353103bdd..0111bd904 100644 --- a/api-error-response_test.go +++ b/api-error-response_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-get-lifecycle.go b/api-get-lifecycle.go index 8097bfc02..96ccfe902 100644 --- a/api-get-lifecycle.go +++ b/api-get-lifecycle.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-get-object-acl.go b/api-get-object-acl.go index af5544da3..ea809492a 100644 --- a/api-get-object-acl.go +++ b/api-get-object-acl.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2018 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-get-object-context.go b/api-get-object-context.go index f8dfac7d6..3c3afd059 100644 --- a/api-get-object-context.go +++ b/api-get-object-context.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-get-object-file.go b/api-get-object-file.go index a852220a2..98837bfa3 100644 --- a/api-get-object-file.go +++ b/api-get-object-file.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-get-object.go b/api-get-object.go index 0bf556ec6..a00ddaa89 100644 --- a/api-get-object.go +++ b/api-get-object.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-get-options.go b/api-get-options.go index dbf062d61..323b1c909 100644 --- a/api-get-options.go +++ b/api-get-options.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-get-policy.go b/api-get-policy.go index 12d4c590e..035685d04 100644 --- a/api-get-policy.go +++ b/api-get-policy.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-list.go b/api-list.go index 2f1350a34..6c4259eb4 100644 --- a/api-list.go +++ b/api-list.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-notification.go b/api-notification.go index 1c01e362b..38a3f992f 100644 --- a/api-notification.go +++ b/api-notification.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-presigned.go b/api-presigned.go index a2c060786..8ffcdd712 100644 --- a/api-presigned.go +++ b/api-presigned.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-put-bucket.go b/api-put-bucket.go index 33dc0cf3d..69d33e315 100644 --- a/api-put-bucket.go +++ b/api-put-bucket.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-put-object-common.go b/api-put-object-common.go index ce3396e7d..8484a7854 100644 --- a/api-put-object-common.go +++ b/api-put-object-common.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-put-object-context.go b/api-put-object-context.go index ff4663e2f..415a7878f 100644 --- a/api-put-object-context.go +++ b/api-put-object-context.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-put-object-copy.go b/api-put-object-copy.go index 21322ef6a..c6ecf39d8 100644 --- a/api-put-object-copy.go +++ b/api-put-object-copy.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017, 2018 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017, 2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-put-object-file-context.go b/api-put-object-file-context.go index 140a9c069..3574a6b5b 100644 --- a/api-put-object-file-context.go +++ b/api-put-object-file-context.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-put-object-file.go b/api-put-object-file.go index 7c8e05117..23df6cd1c 100644 --- a/api-put-object-file.go +++ b/api-put-object-file.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-put-object-multipart.go b/api-put-object-multipart.go index db92520e8..62f43efec 100644 --- a/api-put-object-multipart.go +++ b/api-put-object-multipart.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-put-object-streaming.go b/api-put-object-streaming.go index 211d1c23c..1190955ad 100644 --- a/api-put-object-streaming.go +++ b/api-put-object-streaming.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-put-object.go b/api-put-object.go index 0330cd99d..cf893c539 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-put-object_test.go b/api-put-object_test.go index d96abab9a..2867f3a19 100644 --- a/api-put-object_test.go +++ b/api-put-object_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-remove.go b/api-remove.go index f33df4dfc..29f6f347e 100644 --- a/api-remove.go +++ b/api-remove.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-s3-datatypes.go b/api-s3-datatypes.go index 8d8880c05..a6b125522 100644 --- a/api-s3-datatypes.go +++ b/api-s3-datatypes.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-select.go b/api-select.go index 71c5838b3..0b5762450 100644 --- a/api-select.go +++ b/api-select.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * (C) 2018 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-stat.go b/api-stat.go index f04bac98a..15d0af39e 100644 --- a/api-stat.go +++ b/api-stat.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api.go b/api.go index 40a5d881b..8f4fbbfd0 100644 --- a/api.go +++ b/api.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2018 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -108,9 +108,9 @@ const ( // User Agent should always following the below style. // Please open an issue to discuss any new changes here. // -// Minio (OS; ARCH) LIB/VER APP/VER +// MinIO (OS; ARCH) LIB/VER APP/VER const ( - libraryUserAgentPrefix = "Minio (" + runtime.GOOS + "; " + runtime.GOARCH + ") " + libraryUserAgentPrefix = "MinIO (" + runtime.GOOS + "; " + runtime.GOARCH + ") " libraryUserAgent = libraryUserAgentPrefix + libraryName + "/" + libraryVersion ) diff --git a/api_unit_test.go b/api_unit_test.go index 35b01797e..cfd78c95d 100644 --- a/api_unit_test.go +++ b/api_unit_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -188,7 +188,7 @@ func TestMakeTargetURL(t *testing.T) { } for i, testCase := range testCases { - // Initialize a Minio client + // Initialize a MinIO client c, _ := New(testCase.addr, "foo", "bar", testCase.secure) isVirtualHost := c.isVirtualHostStyleRequest(*c.endpointURL, testCase.bucketName) u, err := c.makeTargetURL(testCase.bucketName, testCase.objectName, testCase.bucketLocation, isVirtualHost, testCase.queryValues) diff --git a/bucket-cache.go b/bucket-cache.go index 67709cae8..4ea28e569 100644 --- a/bucket-cache.go +++ b/bucket-cache.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/bucket-cache_test.go b/bucket-cache_test.go index 507b40862..477468ea2 100644 --- a/bucket-cache_test.go +++ b/bucket-cache_test.go @@ -1,6 +1,6 @@ /* * Copyright - * 2015, 2016, 2017 Minio, Inc. + * 2015, 2016, 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -156,10 +156,10 @@ func TestGetBucketLocationRequest(t *testing.T) { {"storage.googleapis.com", "my-access-key", "my-secret-key", false}, {"storage.googleapis.com", "", "my-secret-key", false}, - // endpoint custom domain running Minio server. - {"play.minio.io", "", "", false}, - {"play.minio.io", "my-access-key", "my-secret-key", false}, - {"play.minio.io", "my-acess-key", "", false}, + // endpoint custom domain running MinIO server. + {"play.min.io", "", "", false}, + {"play.min.io", "my-access-key", "my-secret-key", false}, + {"play.min.io", "my-acess-key", "", false}, } testCases := []struct { bucketName string diff --git a/bucket-notification.go b/bucket-notification.go index ea303dd9d..11fc299ff 100644 --- a/bucket-notification.go +++ b/bucket-notification.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/constants.go b/constants.go index 737742318..252fc5faf 100644 --- a/constants.go +++ b/constants.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/core.go b/core.go index 4d51363f0..afd4139ee 100644 --- a/core.go +++ b/core.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/core_test.go b/core_test.go index 367a1cffa..525d6f6d7 100644 --- a/core_test.go +++ b/core_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -86,7 +86,7 @@ func TestGetObjectCore(t *testing.T) { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") @@ -262,7 +262,7 @@ func TestGetObjectContentEncoding(t *testing.T) { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") @@ -339,7 +339,7 @@ func TestGetBucketPolicy(t *testing.T) { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") @@ -402,7 +402,7 @@ func TestCoreCopyObject(t *testing.T) { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") @@ -516,7 +516,7 @@ func TestCoreCopyObjectPart(t *testing.T) { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") @@ -661,7 +661,7 @@ func TestCorePutObject(t *testing.T) { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") diff --git a/docs/API.md b/docs/API.md index a4c8d7d7b..b868443e8 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1,8 +1,8 @@ -# Minio Go Client API Reference [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) +# MinIO Go Client API Reference [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) -## Initialize Minio Client object. +## Initialize MinIO Client object. -## Minio +## MinIO ```go package main @@ -18,7 +18,7 @@ func main() { ssl := true // Initialize minio client object. - minioClient, err := minio.New("play.minio.io:9000", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", ssl) + minioClient, err := minio.New("play.min.io:9000", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", ssl) if err != nil { fmt.Println(err) return @@ -71,7 +71,7 @@ func main() { | | [`RemoveObjectsWithContext`](#RemoveObjectsWithContext) | | | | | | [`SelectObjectContent`](#SelectObjectContent) | | ## 1. Constructor - + ### New(endpoint, accessKeyID, secretAccessKey string, ssl bool) (*Client, error) Initializes a new client object. @@ -553,7 +553,7 @@ __minio.PutObjectOptions__ | `opts.ContentLanguage` | _string_ | Content language of object, e.g "French" | | `opts.CacheControl` | _string_ | Used to specify directives for caching mechanisms in both requests and responses e.g "max-age=600"| | `opts.ServerSideEncryption` | _encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go) | -| `opts.StorageClass` | _string_ | Specify storage class for the object. Supported values for Minio server are `REDUCED_REDUNDANCY` and `STANDARD` | +| `opts.StorageClass` | _string_ | Specify storage class for the object. Supported values for MinIO server are `REDUCED_REDUNDANCY` and `STANDARD` | | `opts.WebsiteRedirectLocation` | _string_ | Specify a redirect for the object, to another object in the same bucket or to a external URL. | __Example__ diff --git a/docs/checker.go.template b/docs/checker.go.template index 2e0f13a53..4e332da42 100644 --- a/docs/checker.go.template +++ b/docs/checker.go.template @@ -11,7 +11,7 @@ func main() { ssl := true // Initialize minio client object. - minioClient, err := minio.New("play.minio.io:9000", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", ssl) + minioClient, err := minio.New("play.min.io:9000", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", ssl) if err != nil { fmt.Println(err) return diff --git a/docs/validator.go b/docs/validator.go index 7d5cbaaab..e75de75bd 100644 --- a/docs/validator.go +++ b/docs/validator.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -199,7 +199,7 @@ func main() { app.HideVersion = true app.HideHelpCommand = true app.Usage = "Validates code block sections inside API.md" - app.Author = "Minio.io" + app.Author = "min.io" app.Flags = globalFlags // Help template for validator app.CustomAppHelpTemplate = `NAME: diff --git a/docs/zh_CN/API.md b/docs/zh_CN/API.md index dacdf4b44..583892f6a 100644 --- a/docs/zh_CN/API.md +++ b/docs/zh_CN/API.md @@ -1,8 +1,8 @@ -# Minio Go Client API文档 [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) +# MinIO Go Client API文档 [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) -## 初使化Minio Client对象。 +## 初使化MinIO Client对象。 -## Minio +## MinIO ```go package main @@ -18,7 +18,7 @@ func main() { ssl := true // 初使化minio client对象。 - minioClient, err := minio.New("play.minio.io:9000", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", ssl) + minioClient, err := minio.New("play.min.io:9000", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", ssl) if err != nil { fmt.Println(err) return @@ -69,7 +69,7 @@ func main() { | | [`FPutObjectWithContext`](#FPutObjectWithContext) | | | | | | [`FGetObjectWithContext`](#FGetObjectWithContext) | | | | ## 1. 构造函数 - + ### New(endpoint, accessKeyID, secretAccessKey string, ssl bool) (*Client, error) 初使化一个新的client对象。 diff --git a/examples/minio/listenbucketnotification.go b/examples/minio/listenbucketnotification.go index 4c48510da..eff0219a6 100644 --- a/examples/minio/listenbucketnotification.go +++ b/examples/minio/listenbucketnotification.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -34,7 +34,7 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - minioClient, err := minio.New("play.minio.io:9000", "YOUR-ACCESS", "YOUR-SECRET", true) + minioClient, err := minio.New("play.min.io:9000", "YOUR-ACCESS", "YOUR-SECRET", true) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/bucketexists.go b/examples/s3/bucketexists.go index 20dea30a3..84a7595f5 100644 --- a/examples/s3/bucketexists.go +++ b/examples/s3/bucketexists.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/composeobject.go b/examples/s3/composeobject.go index 2f76ff053..1966383ce 100644 --- a/examples/s3/composeobject.go +++ b/examples/s3/composeobject.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/copyobject.go b/examples/s3/copyobject.go index a7c3eca45..9dcfab168 100644 --- a/examples/s3/copyobject.go +++ b/examples/s3/copyobject.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/fgetobject-context.go b/examples/s3/fgetobject-context.go index 6004baa14..13acc3ce7 100644 --- a/examples/s3/fgetobject-context.go +++ b/examples/s3/fgetobject-context.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/fgetobject.go b/examples/s3/fgetobject.go index 819a34f91..b747a907f 100644 --- a/examples/s3/fgetobject.go +++ b/examples/s3/fgetobject.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/fputencrypted-object.go b/examples/s3/fputencrypted-object.go index 5da9f9d71..b0dc24006 100644 --- a/examples/s3/fputencrypted-object.go +++ b/examples/s3/fputencrypted-object.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/fputobject-context.go b/examples/s3/fputobject-context.go index d7c941c2b..1cd030c5a 100644 --- a/examples/s3/fputobject-context.go +++ b/examples/s3/fputobject-context.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/fputobject.go b/examples/s3/fputobject.go index 34d876804..6ce0de68f 100644 --- a/examples/s3/fputobject.go +++ b/examples/s3/fputobject.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/get-encrypted-object.go b/examples/s3/get-encrypted-object.go index 62a06d59f..43be44e6b 100644 --- a/examples/s3/get-encrypted-object.go +++ b/examples/s3/get-encrypted-object.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/getbucketlifecycle.go b/examples/s3/getbucketlifecycle.go index 2e3ef419e..3086ae7b7 100644 --- a/examples/s3/getbucketlifecycle.go +++ b/examples/s3/getbucketlifecycle.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/getbucketnotification.go b/examples/s3/getbucketnotification.go index 19349baaf..9210dc5a0 100644 --- a/examples/s3/getbucketnotification.go +++ b/examples/s3/getbucketnotification.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/getbucketpolicy.go b/examples/s3/getbucketpolicy.go index e5b594057..e8646f874 100644 --- a/examples/s3/getbucketpolicy.go +++ b/examples/s3/getbucketpolicy.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/getobject-client-encryption.go b/examples/s3/getobject-client-encryption.go index 6b06073c9..6b06dca81 100644 --- a/examples/s3/getobject-client-encryption.go +++ b/examples/s3/getobject-client-encryption.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2018 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/getobject-context.go b/examples/s3/getobject-context.go index c7d41707a..4e68daa5b 100644 --- a/examples/s3/getobject-context.go +++ b/examples/s3/getobject-context.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/getobject.go b/examples/s3/getobject.go index e17ef8172..e09401aa7 100644 --- a/examples/s3/getobject.go +++ b/examples/s3/getobject.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/getobjectacl.go b/examples/s3/getobjectacl.go index f2bbd95d5..b397fa45f 100644 --- a/examples/s3/getobjectacl.go +++ b/examples/s3/getobjectacl.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2018-2019 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2018-2019 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/listbuckets.go b/examples/s3/listbuckets.go index 5eae587b4..fa1861d54 100644 --- a/examples/s3/listbuckets.go +++ b/examples/s3/listbuckets.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/listincompleteuploads.go b/examples/s3/listincompleteuploads.go index a5a79b603..1fe4b2ac9 100644 --- a/examples/s3/listincompleteuploads.go +++ b/examples/s3/listincompleteuploads.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/listobjects-N.go b/examples/s3/listobjects-N.go index 55bceb470..86e6df599 100644 --- a/examples/s3/listobjects-N.go +++ b/examples/s3/listobjects-N.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/listobjects.go b/examples/s3/listobjects.go index 1da2e3faa..5c9d0e94b 100644 --- a/examples/s3/listobjects.go +++ b/examples/s3/listobjects.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/listobjectsV2.go b/examples/s3/listobjectsV2.go index 190aec36b..2750a82a9 100644 --- a/examples/s3/listobjectsV2.go +++ b/examples/s3/listobjectsV2.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/makebucket.go b/examples/s3/makebucket.go index 419c96cf2..fe7a9a5c6 100644 --- a/examples/s3/makebucket.go +++ b/examples/s3/makebucket.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/presignedgetobject.go b/examples/s3/presignedgetobject.go index fd7fb9e8d..0a4c797ea 100644 --- a/examples/s3/presignedgetobject.go +++ b/examples/s3/presignedgetobject.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/presignedheadobject.go b/examples/s3/presignedheadobject.go index 8dbc0a4b7..d9354550b 100644 --- a/examples/s3/presignedheadobject.go +++ b/examples/s3/presignedheadobject.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/presignedpostpolicy.go b/examples/s3/presignedpostpolicy.go index 205ac95a3..af6f6f106 100644 --- a/examples/s3/presignedpostpolicy.go +++ b/examples/s3/presignedpostpolicy.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/presignedputobject.go b/examples/s3/presignedputobject.go index b2f8b4f82..516a3b01e 100644 --- a/examples/s3/presignedputobject.go +++ b/examples/s3/presignedputobject.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/put-encrypted-object.go b/examples/s3/put-encrypted-object.go index 48b93671e..d15d0190b 100644 --- a/examples/s3/put-encrypted-object.go +++ b/examples/s3/put-encrypted-object.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/putobject-client-encryption.go b/examples/s3/putobject-client-encryption.go index 77d83b400..b46b3476f 100644 --- a/examples/s3/putobject-client-encryption.go +++ b/examples/s3/putobject-client-encryption.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2018 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/putobject-context.go b/examples/s3/putobject-context.go index acc923f7e..1c40372d6 100644 --- a/examples/s3/putobject-context.go +++ b/examples/s3/putobject-context.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/putobject-getobject-sse.go b/examples/s3/putobject-getobject-sse.go index 4e459b5d7..1415d6137 100644 --- a/examples/s3/putobject-getobject-sse.go +++ b/examples/s3/putobject-getobject-sse.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/putobject-progress.go b/examples/s3/putobject-progress.go index 0e92dd65e..fc034c4b0 100644 --- a/examples/s3/putobject-progress.go +++ b/examples/s3/putobject-progress.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/putobject-s3-accelerate.go b/examples/s3/putobject-s3-accelerate.go index 06345cd87..5203049c9 100644 --- a/examples/s3/putobject-s3-accelerate.go +++ b/examples/s3/putobject-s3-accelerate.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/putobject-streaming.go b/examples/s3/putobject-streaming.go index 85b78dd45..bf7c5cc62 100644 --- a/examples/s3/putobject-streaming.go +++ b/examples/s3/putobject-streaming.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/putobject.go b/examples/s3/putobject.go index b9e4ff16c..dd4a024d0 100644 --- a/examples/s3/putobject.go +++ b/examples/s3/putobject.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/removeallbucketnotification.go b/examples/s3/removeallbucketnotification.go index 1186afad8..9149bc133 100644 --- a/examples/s3/removeallbucketnotification.go +++ b/examples/s3/removeallbucketnotification.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/removebucket.go b/examples/s3/removebucket.go index 7a7737ee0..066eb1f7c 100644 --- a/examples/s3/removebucket.go +++ b/examples/s3/removebucket.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/removeincompleteupload.go b/examples/s3/removeincompleteupload.go index 31cc8790b..efc768fa3 100644 --- a/examples/s3/removeincompleteupload.go +++ b/examples/s3/removeincompleteupload.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/removeobject.go b/examples/s3/removeobject.go index 7e5848576..cdebd7190 100644 --- a/examples/s3/removeobject.go +++ b/examples/s3/removeobject.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/removeobjects.go b/examples/s3/removeobjects.go index a58113483..007c146b0 100644 --- a/examples/s3/removeobjects.go +++ b/examples/s3/removeobjects.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/selectobject.go b/examples/s3/selectobject.go index e23ccf8e1..a6985fc74 100644 --- a/examples/s3/selectobject.go +++ b/examples/s3/selectobject.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2018 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/setbucketlifecycle.go b/examples/s3/setbucketlifecycle.go index 7eaa946f7..b569e7403 100644 --- a/examples/s3/setbucketlifecycle.go +++ b/examples/s3/setbucketlifecycle.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/setbucketnotification.go b/examples/s3/setbucketnotification.go index b5af30f06..5e644644e 100644 --- a/examples/s3/setbucketnotification.go +++ b/examples/s3/setbucketnotification.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/setbucketpolicy.go b/examples/s3/setbucketpolicy.go index bc42da826..8ab169799 100644 --- a/examples/s3/setbucketpolicy.go +++ b/examples/s3/setbucketpolicy.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/s3/statobject.go b/examples/s3/statobject.go index 0b27a83b3..b20a84199 100644 --- a/examples/s3/statobject.go +++ b/examples/s3/statobject.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/functional_tests.go b/functional_tests.go index cfb05e18b..b412edb23 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -184,9 +184,9 @@ func isErrNotImplemented(err error) bool { func init() { // If server endpoint is not set, all tests default to - // using https://play.minio.io:9000 + // using https://play.min.io:9000 if os.Getenv(serverEndpoint) == "" { - os.Setenv(serverEndpoint, "play.minio.io:9000") + os.Setenv(serverEndpoint, "play.min.io:9000") os.Setenv(accessKey, "Q3AM3UQ867SPQQA43P2F") os.Setenv(secretKey, "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG") os.Setenv(enableHTTPS, "1") @@ -307,7 +307,7 @@ func testMakeBucketError() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client creation failed", err) return } @@ -315,7 +315,7 @@ func testMakeBucketError() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -363,10 +363,10 @@ func testMetadataSizeLimit() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client creation failed", err) return } - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName @@ -443,7 +443,7 @@ func testMakeBucketRegions() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client creation failed", err) return } @@ -451,7 +451,7 @@ func testMakeBucketRegions() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -510,7 +510,7 @@ func testPutObjectReadAt() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -518,7 +518,7 @@ func testPutObjectReadAt() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -620,7 +620,7 @@ func testPutObjectWithMetadata() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -628,7 +628,7 @@ func testPutObjectWithMetadata() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -731,7 +731,7 @@ func testPutObjectWithContentLanguage() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -739,7 +739,7 @@ func testPutObjectWithContentLanguage() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -810,7 +810,7 @@ func testPutObjectStreaming() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -818,7 +818,7 @@ func testPutObjectStreaming() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -875,7 +875,7 @@ func testGetObjectSeekEnd() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -883,7 +883,7 @@ func testGetObjectSeekEnd() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -1007,7 +1007,7 @@ func testGetObjectClosedTwice() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -1015,7 +1015,7 @@ func testGetObjectClosedTwice() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -1103,12 +1103,12 @@ func testRemoveObjectsWithContext() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Enable tracing, write to stdout. // c.TraceOn(os.Stderr) @@ -1199,12 +1199,12 @@ func testRemoveMultipleObjects() { ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Enable tracing, write to stdout. // c.TraceOn(os.Stderr) @@ -1286,7 +1286,7 @@ func testFPutObjectMultipart() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -1294,7 +1294,7 @@ func testFPutObjectMultipart() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -1400,7 +1400,7 @@ func testFPutObject() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -1408,7 +1408,7 @@ func testFPutObject() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -1586,7 +1586,7 @@ func testFPutObjectWithContext() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -1594,7 +1594,7 @@ func testFPutObjectWithContext() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -1697,7 +1697,7 @@ func testFPutObjectWithContextV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -1705,7 +1705,7 @@ func testFPutObjectWithContextV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -1808,7 +1808,7 @@ func testPutObjectWithContext() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -1816,7 +1816,7 @@ func testPutObjectWithContext() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Make a new bucket. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -1885,7 +1885,7 @@ func testGetObjectReadSeekFunctional() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -1893,7 +1893,7 @@ func testGetObjectReadSeekFunctional() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -2060,7 +2060,7 @@ func testGetObjectReadAtFunctional() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -2068,7 +2068,7 @@ func testGetObjectReadAtFunctional() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -2247,7 +2247,7 @@ func testPresignedPostPolicy() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -2255,7 +2255,7 @@ func testPresignedPostPolicy() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -2447,7 +2447,7 @@ func testCopyObject() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -2455,7 +2455,7 @@ func testCopyObject() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -2677,7 +2677,7 @@ func testSSECEncryptedGetObjectReadSeekFunctional() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -2685,7 +2685,7 @@ func testSSECEncryptedGetObjectReadSeekFunctional() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -2865,7 +2865,7 @@ func testSSES3EncryptedGetObjectReadSeekFunctional() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -2873,7 +2873,7 @@ func testSSES3EncryptedGetObjectReadSeekFunctional() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -3051,7 +3051,7 @@ func testSSECEncryptedGetObjectReadAtFunctional() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -3059,7 +3059,7 @@ func testSSECEncryptedGetObjectReadAtFunctional() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -3242,7 +3242,7 @@ func testSSES3EncryptedGetObjectReadAtFunctional() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -3250,7 +3250,7 @@ func testSSES3EncryptedGetObjectReadAtFunctional() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -3434,7 +3434,7 @@ func testSSECEncryptionPutGet() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -3442,7 +3442,7 @@ func testSSECEncryptionPutGet() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -3548,7 +3548,7 @@ func testSSECEncryptionFPut() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -3556,7 +3556,7 @@ func testSSECEncryptionFPut() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -3678,7 +3678,7 @@ func testSSES3EncryptionPutGet() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -3686,7 +3686,7 @@ func testSSES3EncryptionPutGet() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -3790,7 +3790,7 @@ func testSSES3EncryptionFPut() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -3798,7 +3798,7 @@ func testSSES3EncryptionFPut() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -3925,7 +3925,7 @@ func testBucketNotification() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -3933,7 +3933,7 @@ func testBucketNotification() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") bucketName := os.Getenv("NOTIFY_BUCKET") args["bucketName"] = bucketName @@ -4021,7 +4021,7 @@ func testFunctional() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, nil, startTime, "", "Minio client object creation failed", err) + logError(testName, function, nil, startTime, "", "MinIO client object creation failed", err) return } @@ -4029,7 +4029,7 @@ func testFunctional() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -4636,7 +4636,7 @@ func testGetObjectModified() { ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -4644,7 +4644,7 @@ func testGetObjectModified() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Make a new bucket. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -4736,7 +4736,7 @@ func testPutObjectUploadSeekedObject() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -4744,7 +4744,7 @@ func testPutObjectUploadSeekedObject() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Make a new bucket. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -4869,7 +4869,7 @@ func testMakeBucketErrorV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } @@ -4877,7 +4877,7 @@ func testMakeBucketErrorV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -4930,7 +4930,7 @@ func testGetObjectClosedTwiceV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } @@ -4938,7 +4938,7 @@ func testGetObjectClosedTwiceV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -5030,7 +5030,7 @@ func testFPutObjectV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } @@ -5038,7 +5038,7 @@ func testFPutObjectV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -5201,7 +5201,7 @@ func testMakeBucketRegionsV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } @@ -5209,7 +5209,7 @@ func testMakeBucketRegionsV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -5264,7 +5264,7 @@ func testGetObjectReadSeekFunctionalV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } @@ -5272,7 +5272,7 @@ func testGetObjectReadSeekFunctionalV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -5428,7 +5428,7 @@ func testGetObjectReadAtFunctionalV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } @@ -5436,7 +5436,7 @@ func testGetObjectReadAtFunctionalV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -5598,7 +5598,7 @@ func testCopyObjectV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } @@ -5606,7 +5606,7 @@ func testCopyObjectV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -5865,7 +5865,7 @@ func testComposeObjectErrorCasesV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } @@ -5960,7 +5960,7 @@ func testCompose10KSourcesV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } @@ -5982,7 +5982,7 @@ func testEncryptedEmptyObject() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } @@ -6234,7 +6234,7 @@ func testUnencryptedToSSECCopyObject() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. @@ -6262,7 +6262,7 @@ func testUnencryptedToSSES3CopyObject() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. @@ -6290,7 +6290,7 @@ func testUnencryptedToUnencryptedCopyObject() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. @@ -6317,7 +6317,7 @@ func testEncryptedSSECToSSECCopyObject() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. @@ -6345,7 +6345,7 @@ func testEncryptedSSECToSSES3CopyObject() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. @@ -6373,7 +6373,7 @@ func testEncryptedSSECToUnencryptedCopyObject() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. @@ -6401,7 +6401,7 @@ func testEncryptedSSES3ToSSECCopyObject() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. @@ -6429,7 +6429,7 @@ func testEncryptedSSES3ToSSES3CopyObject() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. @@ -6457,7 +6457,7 @@ func testEncryptedSSES3ToUnencryptedCopyObject() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. @@ -6485,7 +6485,7 @@ func testEncryptedCopyObjectV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. @@ -6512,7 +6512,7 @@ func testDecryptedCopyObject() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } @@ -6566,7 +6566,7 @@ func testSSECMultipartEncryptedToSSECCopyObjectPart() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } @@ -6577,7 +6577,7 @@ func testSSECMultipartEncryptedToSSECCopyObjectPart() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") @@ -6738,7 +6738,7 @@ func testSSECEncryptedToSSECCopyObjectPart() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } @@ -6749,7 +6749,7 @@ func testSSECEncryptedToSSECCopyObjectPart() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") @@ -6889,7 +6889,7 @@ func testSSECEncryptedToUnencryptedCopyPart() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } @@ -6900,7 +6900,7 @@ func testSSECEncryptedToUnencryptedCopyPart() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") @@ -7039,7 +7039,7 @@ func testSSECEncryptedToSSES3CopyObjectPart() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } @@ -7050,7 +7050,7 @@ func testSSECEncryptedToSSES3CopyObjectPart() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") @@ -7191,7 +7191,7 @@ func testUnencryptedToSSECCopyObjectPart() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } @@ -7202,7 +7202,7 @@ func testUnencryptedToSSECCopyObjectPart() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") @@ -7340,7 +7340,7 @@ func testUnencryptedToUnencryptedCopyPart() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } @@ -7351,7 +7351,7 @@ func testUnencryptedToUnencryptedCopyPart() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") @@ -7486,7 +7486,7 @@ func testUnencryptedToSSES3CopyObjectPart() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } @@ -7497,7 +7497,7 @@ func testUnencryptedToSSES3CopyObjectPart() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") @@ -7635,7 +7635,7 @@ func testSSES3EncryptedToSSECCopyObjectPart() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } @@ -7646,7 +7646,7 @@ func testSSES3EncryptedToSSECCopyObjectPart() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") @@ -7784,7 +7784,7 @@ func testSSES3EncryptedToUnencryptedCopyPart() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } @@ -7795,7 +7795,7 @@ func testSSES3EncryptedToUnencryptedCopyPart() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") @@ -7931,7 +7931,7 @@ func testSSES3EncryptedToSSES3CopyObjectPart() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } @@ -7942,7 +7942,7 @@ func testSSES3EncryptedToSSES3CopyObjectPart() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") @@ -8079,7 +8079,7 @@ func testUserMetadataCopying() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -8266,7 +8266,7 @@ func testUserMetadataCopyingV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return } @@ -8289,7 +8289,7 @@ func testStorageClassMetadataPutObject() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } @@ -8379,7 +8379,7 @@ func testStorageClassInvalidMetadataPutObject() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } @@ -8424,7 +8424,7 @@ func testStorageClassMetadataCopyObject() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } @@ -8537,7 +8537,7 @@ func testPutObjectNoLengthV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return } @@ -8545,7 +8545,7 @@ func testPutObjectNoLengthV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -8611,7 +8611,7 @@ func testPutObjectsUnknownV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return } @@ -8619,7 +8619,7 @@ func testPutObjectsUnknownV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -8695,7 +8695,7 @@ func testPutObject0ByteV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return } @@ -8703,7 +8703,7 @@ func testPutObject0ByteV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -8757,7 +8757,7 @@ func testComposeObjectErrorCases() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -8780,7 +8780,7 @@ func testCompose10KSources() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -8806,7 +8806,7 @@ func testFunctionalV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return } @@ -8814,7 +8814,7 @@ func testFunctionalV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -9219,7 +9219,7 @@ func testGetObjectWithContext() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client v4 object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } @@ -9227,7 +9227,7 @@ func testGetObjectWithContext() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -9327,7 +9327,7 @@ func testFGetObjectWithContext() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client v4 object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } @@ -9335,7 +9335,7 @@ func testFGetObjectWithContext() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -9423,7 +9423,7 @@ func testGetObjectACL() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client v4 object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } @@ -9431,7 +9431,7 @@ func testGetObjectACL() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -9578,7 +9578,7 @@ func testPutObjectWithContextV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return } @@ -9586,7 +9586,7 @@ func testPutObjectWithContextV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Make a new bucket. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -9660,7 +9660,7 @@ func testGetObjectWithContextV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return } @@ -9668,7 +9668,7 @@ func testGetObjectWithContextV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -9766,7 +9766,7 @@ func testFGetObjectWithContextV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return } @@ -9774,7 +9774,7 @@ func testFGetObjectWithContextV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -9859,7 +9859,7 @@ func testListObjects() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client v4 object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } @@ -9867,7 +9867,7 @@ func testListObjects() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") diff --git a/get-options_test.go b/get-options_test.go index c5344a0c6..a447f09d0 100644 --- a/get-options_test.go +++ b/get-options_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/hook-reader.go b/hook-reader.go index f23aec754..f251c1e95 100644 --- a/hook-reader.go +++ b/hook-reader.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/chain.go b/pkg/credentials/chain.go index e29826f48..6dc8e9d05 100644 --- a/pkg/credentials/chain.go +++ b/pkg/credentials/chain.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/chain_test.go b/pkg/credentials/chain_test.go index d26e376ff..2e18c29f2 100644 --- a/pkg/credentials/chain_test.go +++ b/pkg/credentials/chain_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/config.json.sample b/pkg/credentials/config.json.sample index 130746f4b..0affa58cf 100644 --- a/pkg/credentials/config.json.sample +++ b/pkg/credentials/config.json.sample @@ -2,7 +2,7 @@ "version": "8", "hosts": { "play": { - "url": "https://play.minio.io:9000", + "url": "https://play.min.io:9000", "accessKey": "Q3AM3UQ867SPQQA43P2F", "secretKey": "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", "api": "S3v2" diff --git a/pkg/credentials/credentials.go b/pkg/credentials/credentials.go index 4bfdad413..1a48751b5 100644 --- a/pkg/credentials/credentials.go +++ b/pkg/credentials/credentials.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/credentials_test.go b/pkg/credentials/credentials_test.go index 92c77c4cb..57ea67983 100644 --- a/pkg/credentials/credentials_test.go +++ b/pkg/credentials/credentials_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/doc.go b/pkg/credentials/doc.go index c48784ba8..0c94477b7 100644 --- a/pkg/credentials/doc.go +++ b/pkg/credentials/doc.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/env_aws.go b/pkg/credentials/env_aws.go index f9b2cc33a..b6e60d0e1 100644 --- a/pkg/credentials/env_aws.go +++ b/pkg/credentials/env_aws.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/env_minio.go b/pkg/credentials/env_minio.go index d72e77185..5f1ae0d25 100644 --- a/pkg/credentials/env_minio.go +++ b/pkg/credentials/env_minio.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/env_test.go b/pkg/credentials/env_test.go index 09cd77f7a..2f982ae90 100644 --- a/pkg/credentials/env_test.go +++ b/pkg/credentials/env_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/file_aws_credentials.go b/pkg/credentials/file_aws_credentials.go index d61f197db..ff07bc55b 100644 --- a/pkg/credentials/file_aws_credentials.go +++ b/pkg/credentials/file_aws_credentials.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/file_minio_client.go b/pkg/credentials/file_minio_client.go index 6a6827e37..117ceb6ef 100644 --- a/pkg/credentials/file_minio_client.go +++ b/pkg/credentials/file_minio_client.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -40,7 +40,7 @@ type FileMinioClient struct { // Windows: "%USERALIAS%\mc\config.json" filename string - // Minio Alias to extract credentials from the shared credentials file. If empty + // MinIO Alias to extract credentials from the shared credentials file. If empty // will default to environment variable "MINIO_ALIAS" or "default" if // environment variable is also not set. alias string diff --git a/pkg/credentials/file_test.go b/pkg/credentials/file_test.go index c85c10494..598c3f52e 100644 --- a/pkg/credentials/file_test.go +++ b/pkg/credentials/file_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/iam_aws.go b/pkg/credentials/iam_aws.go index 05b2a8bb4..310785209 100644 --- a/pkg/credentials/iam_aws.go +++ b/pkg/credentials/iam_aws.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/iam_aws_test.go b/pkg/credentials/iam_aws_test.go index 4dbbb0a13..cc5518352 100644 --- a/pkg/credentials/iam_aws_test.go +++ b/pkg/credentials/iam_aws_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/signature-type.go b/pkg/credentials/signature-type.go index 1b768e8c3..b79433305 100644 --- a/pkg/credentials/signature-type.go +++ b/pkg/credentials/signature-type.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/static.go b/pkg/credentials/static.go index 8b0ba711c..7dde00b0a 100644 --- a/pkg/credentials/static.go +++ b/pkg/credentials/static.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/static_test.go b/pkg/credentials/static_test.go index f1d2d856c..65bec0565 100644 --- a/pkg/credentials/static_test.go +++ b/pkg/credentials/static_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/sts_client_grants.go b/pkg/credentials/sts_client_grants.go index 0ca307ab8..03134c3d2 100644 --- a/pkg/credentials/sts_client_grants.go +++ b/pkg/credentials/sts_client_grants.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2019 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,7 +43,7 @@ type AssumeRoleWithClientGrantsResponse struct { } // ClientGrantsResult - Contains the response to a successful AssumeRoleWithClientGrants -// request, including temporary credentials that can be used to make Minio API requests. +// request, including temporary credentials that can be used to make MinIO API requests. type ClientGrantsResult struct { AssumedRoleUser AssumedRoleUser `xml:",omitempty"` Audience string `xml:",omitempty"` @@ -64,15 +64,15 @@ type ClientGrantsToken struct { Expiry int } -// A STSClientGrants retrieves credentials from Minio service, and keeps track if +// A STSClientGrants retrieves credentials from MinIO service, and keeps track if // those credentials are expired. type STSClientGrants struct { Expiry - // Required http Client to use when connecting to Minio STS service. + // Required http Client to use when connecting to MinIO STS service. Client *http.Client - // Minio endpoint to fetch STS credentials. + // MinIO endpoint to fetch STS credentials. stsEndpoint string // getClientGrantsTokenExpiry function to retrieve tokens @@ -142,7 +142,7 @@ func getClientGrantsCredentials(clnt *http.Client, endpoint string, return a, nil } -// Retrieve retrieves credentials from the Minio service. +// Retrieve retrieves credentials from the MinIO service. // Error will be returned if the request fails. func (m *STSClientGrants) Retrieve() (Value, error) { a, err := getClientGrantsCredentials(m.Client, m.stsEndpoint, m.getClientGrantsTokenExpiry) diff --git a/pkg/credentials/sts_web_identity.go b/pkg/credentials/sts_web_identity.go index 81a488711..4d53bd236 100644 --- a/pkg/credentials/sts_web_identity.go +++ b/pkg/credentials/sts_web_identity.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2019 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,7 +36,7 @@ type AssumeRoleWithWebIdentityResponse struct { } // WebIdentityResult - Contains the response to a successful AssumeRoleWithWebIdentity -// request, including temporary credentials that can be used to make Minio API requests. +// request, including temporary credentials that can be used to make MinIO API requests. type WebIdentityResult struct { AssumedRoleUser AssumedRoleUser `xml:",omitempty"` Audience string `xml:",omitempty"` @@ -57,15 +57,15 @@ type WebIdentityToken struct { Expiry int } -// A STSWebIdentity retrieves credentials from Minio service, and keeps track if +// A STSWebIdentity retrieves credentials from MinIO service, and keeps track if // those credentials are expired. type STSWebIdentity struct { Expiry - // Required http Client to use when connecting to Minio STS service. + // Required http Client to use when connecting to MinIO STS service. Client *http.Client - // Minio endpoint to fetch STS credentials. + // MinIO endpoint to fetch STS credentials. stsEndpoint string // getWebIDTokenExpiry function which returns ID tokens @@ -138,7 +138,7 @@ func getWebIdentityCredentials(clnt *http.Client, endpoint string, return a, nil } -// Retrieve retrieves credentials from the Minio service. +// Retrieve retrieves credentials from the MinIO service. // Error will be returned if the request fails. func (m *STSWebIdentity) Retrieve() (Value, error) { a, err := getWebIdentityCredentials(m.Client, m.stsEndpoint, m.getWebIDTokenExpiry) diff --git a/pkg/encrypt/server-side.go b/pkg/encrypt/server-side.go index 2d3c70f00..ac0b69a02 100644 --- a/pkg/encrypt/server-side.go +++ b/pkg/encrypt/server-side.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2018 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/policy/bucket-policy-condition.go b/pkg/policy/bucket-policy-condition.go index 737b810ac..c5eb67238 100644 --- a/pkg/policy/bucket-policy-condition.go +++ b/pkg/policy/bucket-policy-condition.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/policy/bucket-policy-condition_test.go b/pkg/policy/bucket-policy-condition_test.go index 9e4aa8fb6..e1cc2f3d1 100644 --- a/pkg/policy/bucket-policy-condition_test.go +++ b/pkg/policy/bucket-policy-condition_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/policy/bucket-policy.go b/pkg/policy/bucket-policy.go index 79fd8019a..c3dd83d6a 100644 --- a/pkg/policy/bucket-policy.go +++ b/pkg/policy/bucket-policy.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/policy/bucket-policy_test.go b/pkg/policy/bucket-policy_test.go index 1a71d8770..bd3ae59ff 100644 --- a/pkg/policy/bucket-policy_test.go +++ b/pkg/policy/bucket-policy_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/s3signer/request-signature-streaming.go b/pkg/s3signer/request-signature-streaming.go index 156a6d63a..c82e6bd37 100644 --- a/pkg/s3signer/request-signature-streaming.go +++ b/pkg/s3signer/request-signature-streaming.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/s3signer/request-signature-streaming_test.go b/pkg/s3signer/request-signature-streaming_test.go index 297ab97be..e65061e70 100644 --- a/pkg/s3signer/request-signature-streaming_test.go +++ b/pkg/s3signer/request-signature-streaming_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/s3signer/request-signature-v2.go b/pkg/s3signer/request-signature-v2.go index b4070938e..919540690 100644 --- a/pkg/s3signer/request-signature-v2.go +++ b/pkg/s3signer/request-signature-v2.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/s3signer/request-signature-v2_test.go b/pkg/s3signer/request-signature-v2_test.go index 042b6e65c..d94e01252 100644 --- a/pkg/s3signer/request-signature-v2_test.go +++ b/pkg/s3signer/request-signature-v2_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/s3signer/request-signature-v4.go b/pkg/s3signer/request-signature-v4.go index d0ddbfa0e..e23c06baf 100644 --- a/pkg/s3signer/request-signature-v4.go +++ b/pkg/s3signer/request-signature-v4.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/s3signer/request-signature-v4_test.go b/pkg/s3signer/request-signature-v4_test.go index a109a4f2a..d0c9e3095 100644 --- a/pkg/s3signer/request-signature-v4_test.go +++ b/pkg/s3signer/request-signature-v4_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/s3signer/request-signature_test.go b/pkg/s3signer/request-signature_test.go index 75115d19c..3e072cff7 100644 --- a/pkg/s3signer/request-signature_test.go +++ b/pkg/s3signer/request-signature_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/s3signer/test-utils_test.go b/pkg/s3signer/test-utils_test.go index cf96d66c8..8a7d8ff28 100644 --- a/pkg/s3signer/test-utils_test.go +++ b/pkg/s3signer/test-utils_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/s3signer/utils.go b/pkg/s3signer/utils.go index c307c47ee..d07ff6dec 100644 --- a/pkg/s3signer/utils.go +++ b/pkg/s3signer/utils.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/s3signer/utils_test.go b/pkg/s3signer/utils_test.go index a5de14d6e..5ec030712 100644 --- a/pkg/s3signer/utils_test.go +++ b/pkg/s3signer/utils_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/s3utils/utils.go b/pkg/s3utils/utils.go index adceb7f2a..c80cfba7e 100644 --- a/pkg/s3utils/utils.go +++ b/pkg/s3utils/utils.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/s3utils/utils_test.go b/pkg/s3utils/utils_test.go index 55eaaeacf..60f2a06ab 100644 --- a/pkg/s3utils/utils_test.go +++ b/pkg/s3utils/utils_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/set/stringset.go b/pkg/set/stringset.go index efd02629b..e220271bb 100644 --- a/pkg/set/stringset.go +++ b/pkg/set/stringset.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/set/stringset_test.go b/pkg/set/stringset_test.go index d7e6aa799..aeda4e231 100644 --- a/pkg/set/stringset_test.go +++ b/pkg/set/stringset_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/post-policy.go b/post-policy.go index c285fdefd..f9250b2a8 100644 --- a/post-policy.go +++ b/post-policy.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/retry-continous.go b/retry-continous.go index f31dfa6f2..3d25883b0 100644 --- a/retry-continous.go +++ b/retry-continous.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/retry.go b/retry.go index 445167b6a..bd16fbb5a 100644 --- a/retry.go +++ b/retry.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/s3-endpoints.go b/s3-endpoints.go index 0eccd2407..ef4ae74b5 100644 --- a/s3-endpoints.go +++ b/s3-endpoints.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/s3-error.go b/s3-error.go index 3b11776c2..f365157ee 100644 --- a/s3-error.go +++ b/s3-error.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/test-utils_test.go b/test-utils_test.go index 6f6443ccf..88b2fac56 100644 --- a/test-utils_test.go +++ b/test-utils_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/transport.go b/transport.go index d231f8b2a..34efa8980 100644 --- a/transport.go +++ b/transport.go @@ -1,8 +1,8 @@ // +build go1.7 go1.8 /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017-2018 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017-2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/utils.go b/utils.go index 8483f3834..dbad3b103 100644 --- a/utils.go +++ b/utils.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/utils_test.go b/utils_test.go index 2e60f77a2..57ce9d258 100644 --- a/utils_test.go +++ b/utils_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From b6091056d7c591c791974489ca13b8f8ba2b47e9 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 8 Apr 2019 13:24:45 -0700 Subject: [PATCH 015/215] Add configurable PartSize for PutObject (#1093) --- api-put-object-common.go | 25 +++++++++++++++++++++++-- api-put-object-multipart.go | 2 +- api-put-object-streaming.go | 4 ++-- api-put-object.go | 12 +++++++++--- api_unit_test.go | 8 ++++---- 5 files changed, 39 insertions(+), 12 deletions(-) diff --git a/api-put-object-common.go b/api-put-object-common.go index 8484a7854..6a0ff49d6 100644 --- a/api-put-object-common.go +++ b/api-put-object-common.go @@ -67,20 +67,41 @@ func isReadAt(reader io.Reader) (ok bool) { // minPartSize - 64MiB // maxMultipartPutObjectSize - 5TiB // -func optimalPartInfo(objectSize int64) (totalPartsCount int, partSize int64, lastPartSize int64, err error) { +func optimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCount int, partSize int64, lastPartSize int64, err error) { // object size is '-1' set it to 5TiB. if objectSize == -1 { objectSize = maxMultipartPutObjectSize } + // object size is larger than supported maximum. if objectSize > maxMultipartPutObjectSize { err = ErrEntityTooLarge(objectSize, maxMultipartPutObjectSize, "", "") return } + + if int64(configuredPartSize) > objectSize { + err = ErrEntityTooLarge(int64(configuredPartSize), objectSize, "", "") + return + } + + if objectSize > (int64(configuredPartSize) * maxPartsCount) { + err = ErrInvalidArgument("Part size * max_parts(10000) is lesser than input objectSize.") + } + + if configuredPartSize < absMinPartSize { + err = ErrInvalidArgument("Input part size is smaller than allowed minimum of 5MiB.") + return + } + + if configuredPartSize > maxPartSize { + err = ErrInvalidArgument("Input part size is bigger than allowed maximum of 5GiB.") + return + } + // Use floats for part size for all calculations to avoid // overflows during float64 to int64 conversions. partSizeFlt := math.Ceil(float64(objectSize / maxPartsCount)) - partSizeFlt = math.Ceil(partSizeFlt/minPartSize) * minPartSize + partSizeFlt = math.Ceil(partSizeFlt/float64(configuredPartSize)) * float64(configuredPartSize) // Total parts count. totalPartsCount = int(math.Ceil(float64(objectSize) / partSizeFlt)) // Part size. diff --git a/api-put-object-multipart.go b/api-put-object-multipart.go index 62f43efec..95a2599b0 100644 --- a/api-put-object-multipart.go +++ b/api-put-object-multipart.go @@ -73,7 +73,7 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje var complMultipartUpload completeMultipartUpload // Calculate the optimal parts info for a given size. - totalPartsCount, partSize, _, err := optimalPartInfo(-1) + totalPartsCount, partSize, _, err := optimalPartInfo(-1, opts.PartSize) if err != nil { return 0, err } diff --git a/api-put-object-streaming.go b/api-put-object-streaming.go index 1190955ad..0035def3e 100644 --- a/api-put-object-streaming.go +++ b/api-put-object-streaming.go @@ -97,7 +97,7 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa } // Calculate the optimal parts info for a given size. - totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(size) + totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(size, opts.PartSize) if err != nil { return 0, err } @@ -240,7 +240,7 @@ func (c Client) putObjectMultipartStreamNoChecksum(ctx context.Context, bucketNa } // Calculate the optimal parts info for a given size. - totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(size) + totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(size, opts.PartSize) if err != nil { return 0, err } diff --git a/api-put-object.go b/api-put-object.go index cf893c539..06ec8962c 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -44,6 +44,7 @@ type PutObjectOptions struct { NumThreads uint StorageClass string WebsiteRedirectLocation string + PartSize uint64 } // getNumThreads - gets the number of threads to be used in the multipart @@ -147,8 +148,12 @@ func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName stri return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts) } + if opts.PartSize == 0 { + opts.PartSize = minPartSize + } + if c.overrideSignerType.IsV2() { - if size >= 0 && size < minPartSize { + if size >= 0 && size < int64(opts.PartSize) { return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts) } return c.putObjectMultipart(ctx, bucketName, objectName, reader, size, opts) @@ -157,9 +162,10 @@ func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName stri return c.putObjectMultipartStreamNoLength(ctx, bucketName, objectName, reader, opts) } - if size < minPartSize { + if size < int64(opts.PartSize) { return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts) } + // For all sizes greater than 64MiB do multipart. return c.putObjectMultipartStream(ctx, bucketName, objectName, reader, size, opts) } @@ -181,7 +187,7 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName var complMultipartUpload completeMultipartUpload // Calculate the optimal parts info for a given size. - totalPartsCount, partSize, _, err := optimalPartInfo(-1) + totalPartsCount, partSize, _, err := optimalPartInfo(-1, opts.PartSize) if err != nil { return 0, err } diff --git a/api_unit_test.go b/api_unit_test.go index cfd78c95d..00d20f730 100644 --- a/api_unit_test.go +++ b/api_unit_test.go @@ -116,11 +116,11 @@ func TestBucketPolicyTypes(t *testing.T) { // Tests optimal part size. func TestPartSize(t *testing.T) { - _, _, _, err := optimalPartInfo(5000000000000000000) + _, _, _, err := optimalPartInfo(5000000000000000000, minPartSize) if err == nil { t.Fatal("Error: should fail") } - totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(5497558138880) + totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(5497558138880, minPartSize) if err != nil { t.Fatal("Error: ", err) } @@ -133,14 +133,14 @@ func TestPartSize(t *testing.T) { if lastPartSize != 134217728 { t.Fatalf("Error: expecting last part size of 241172480: got %v instead", lastPartSize) } - _, partSize, _, err = optimalPartInfo(5000000000) + _, partSize, _, err = optimalPartInfo(5000000000, minPartSize) if err != nil { t.Fatal("Error:", err) } if partSize != minPartSize { t.Fatalf("Error: expecting part size of %v: got %v instead", minPartSize, partSize) } - totalPartsCount, partSize, lastPartSize, err = optimalPartInfo(-1) + totalPartsCount, partSize, lastPartSize, err = optimalPartInfo(-1, minPartSize) if err != nil { t.Fatal("Error:", err) } From 80c2fe34f1a5d3ade9fc79758f8faafaa149105b Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Wed, 10 Apr 2019 21:46:27 +0000 Subject: [PATCH 016/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 8f4fbbfd0..0a15cb4d5 100644 --- a/api.go +++ b/api.go @@ -102,7 +102,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.22" + libraryVersion = "v6.0.23" ) // User Agent should always following the below style. From a8704b60278f98501c10f694a9c4df8bdd1fac56 Mon Sep 17 00:00:00 2001 From: poornas Date: Mon, 22 Apr 2019 13:51:05 -0700 Subject: [PATCH 017/215] fix optimalPartInfo calculation (#1100) - when client has not configured part size, use optimal part size determined by SDK. Otherwise honor the client specified part size fixes: #1099 --- api-put-object-common.go | 43 +++++++++++++++++++++++----------------- api-put-object.go | 7 ++++--- api_unit_test.go | 41 ++++++++++++++++++++++++++++---------- 3 files changed, 59 insertions(+), 32 deletions(-) diff --git a/api-put-object-common.go b/api-put-object-common.go index 6a0ff49d6..00d98b290 100644 --- a/api-put-object-common.go +++ b/api-put-object-common.go @@ -79,29 +79,36 @@ func optimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCou return } - if int64(configuredPartSize) > objectSize { - err = ErrEntityTooLarge(int64(configuredPartSize), objectSize, "", "") - return - } + var partSizeFlt float64 + if configuredPartSize > 0 { + if int64(configuredPartSize) > objectSize { + err = ErrEntityTooLarge(int64(configuredPartSize), objectSize, "", "") + return + } - if objectSize > (int64(configuredPartSize) * maxPartsCount) { - err = ErrInvalidArgument("Part size * max_parts(10000) is lesser than input objectSize.") - } + if objectSize > (int64(configuredPartSize) * maxPartsCount) { + err = ErrInvalidArgument("Part size * max_parts(10000) is lesser than input objectSize.") + return + } - if configuredPartSize < absMinPartSize { - err = ErrInvalidArgument("Input part size is smaller than allowed minimum of 5MiB.") - return - } + if configuredPartSize < absMinPartSize { + err = ErrInvalidArgument("Input part size is smaller than allowed minimum of 5MiB.") + return + } - if configuredPartSize > maxPartSize { - err = ErrInvalidArgument("Input part size is bigger than allowed maximum of 5GiB.") - return + if configuredPartSize > maxPartSize { + err = ErrInvalidArgument("Input part size is bigger than allowed maximum of 5GiB.") + return + } + partSizeFlt = float64(configuredPartSize) + } else { + configuredPartSize = minPartSize + // Use floats for part size for all calculations to avoid + // overflows during float64 to int64 conversions. + partSizeFlt = math.Ceil(float64(objectSize / maxPartsCount)) + partSizeFlt = math.Ceil(partSizeFlt/float64(configuredPartSize)) * float64(configuredPartSize) } - // Use floats for part size for all calculations to avoid - // overflows during float64 to int64 conversions. - partSizeFlt := math.Ceil(float64(objectSize / maxPartsCount)) - partSizeFlt = math.Ceil(partSizeFlt/float64(configuredPartSize)) * float64(configuredPartSize) // Total parts count. totalPartsCount = int(math.Ceil(float64(objectSize) / partSizeFlt)) // Part size. diff --git a/api-put-object.go b/api-put-object.go index 06ec8962c..4ea9c495a 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -148,12 +148,13 @@ func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName stri return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts) } + partSize := opts.PartSize if opts.PartSize == 0 { - opts.PartSize = minPartSize + partSize = minPartSize } if c.overrideSignerType.IsV2() { - if size >= 0 && size < int64(opts.PartSize) { + if size >= 0 && size < int64(partSize) { return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts) } return c.putObjectMultipart(ctx, bucketName, objectName, reader, size, opts) @@ -162,7 +163,7 @@ func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName stri return c.putObjectMultipartStreamNoLength(ctx, bucketName, objectName, reader, opts) } - if size < int64(opts.PartSize) { + if size < int64(partSize) { return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts) } diff --git a/api_unit_test.go b/api_unit_test.go index 00d20f730..48195325b 100644 --- a/api_unit_test.go +++ b/api_unit_test.go @@ -120,18 +120,31 @@ func TestPartSize(t *testing.T) { if err == nil { t.Fatal("Error: should fail") } - totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(5497558138880, minPartSize) + totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(5243928576, 5*1024*1024) if err != nil { t.Fatal("Error: ", err) } - if totalPartsCount != 9103 { - t.Fatalf("Error: expecting total parts count of 9987: got %v instead", totalPartsCount) + if totalPartsCount != 1001 { + t.Fatalf("Error: expecting total parts count of 1001: got %v instead", totalPartsCount) } - if partSize != 603979776 { - t.Fatalf("Error: expecting part size of 550502400: got %v instead", partSize) + if partSize != 5242880 { + t.Fatalf("Error: expecting part size of 5242880: got %v instead", partSize) } - if lastPartSize != 134217728 { - t.Fatalf("Error: expecting last part size of 241172480: got %v instead", lastPartSize) + if lastPartSize != 1048576 { + t.Fatalf("Error: expecting last part size of 1048576: got %v instead", lastPartSize) + } + totalPartsCount, partSize, lastPartSize, err = optimalPartInfo(5243928576, 0) + if err != nil { + t.Fatal("Error: ", err) + } + if totalPartsCount != 79 { + t.Fatalf("Error: expecting total parts count of 79: got %v instead", totalPartsCount) + } + if partSize != 67108864 { + t.Fatalf("Error: expecting part size of 67108864: got %v instead", partSize) + } + if lastPartSize != 9437184 { + t.Fatalf("Error: expecting last part size of 9437184: got %v instead", lastPartSize) } _, partSize, _, err = optimalPartInfo(5000000000, minPartSize) if err != nil { @@ -140,18 +153,24 @@ func TestPartSize(t *testing.T) { if partSize != minPartSize { t.Fatalf("Error: expecting part size of %v: got %v instead", minPartSize, partSize) } - totalPartsCount, partSize, lastPartSize, err = optimalPartInfo(-1, minPartSize) + // if stream and client configured min part size + _, _, _, err = optimalPartInfo(-1, minPartSize) + if err == nil { + t.Fatal("Error:", err) + } + // if stream and using default optimal part size determined by sdk + totalPartsCount, partSize, lastPartSize, err = optimalPartInfo(-1, 0) if err != nil { t.Fatal("Error:", err) } if totalPartsCount != 9103 { - t.Fatalf("Error: expecting total parts count of 9987: got %v instead", totalPartsCount) + t.Fatalf("Error: expecting total parts count of 9103: got %v instead", totalPartsCount) } if partSize != 603979776 { - t.Fatalf("Error: expecting part size of 550502400: got %v instead", partSize) + t.Fatalf("Error: expecting part size of 603979776: got %v instead", partSize) } if lastPartSize != 134217728 { - t.Fatalf("Error: expecting last part size of 241172480: got %v instead", lastPartSize) + t.Fatalf("Error: expecting last part size of 134217728: got %v instead", lastPartSize) } } From 51138f15dbd9b0d616fdbf7ef19691bb41f7473e Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Wed, 24 Apr 2019 00:31:20 +0000 Subject: [PATCH 018/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 0a15cb4d5..6e329dabf 100644 --- a/api.go +++ b/api.go @@ -102,7 +102,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.23" + libraryVersion = "v6.0.24" ) // User Agent should always following the below style. From 10b3660b8f09a1c25db63da7bf2d4f7449acc366 Mon Sep 17 00:00:00 2001 From: Ashish Kumar Sinha Date: Wed, 1 May 2019 04:57:51 +0530 Subject: [PATCH 019/215] Add check for period and das in bucketname (#1097) The bucket name cannot contain underscores, end with a dash, have consecutive periods, or use dashes adjacent to periods. Ref : https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-s3-bucket-naming-requirements.html --- pkg/s3utils/utils.go | 2 +- pkg/s3utils/utils_test.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/s3utils/utils.go b/pkg/s3utils/utils.go index c80cfba7e..6de0a45aa 100644 --- a/pkg/s3utils/utils.go +++ b/pkg/s3utils/utils.go @@ -282,7 +282,7 @@ func checkBucketNameCommon(bucketName string, strict bool) (err error) { if ipAddress.MatchString(bucketName) { return errors.New("Bucket name cannot be an ip address") } - if strings.Contains(bucketName, "..") { + if strings.Contains(bucketName, "..") || strings.Contains(bucketName, ".-") || strings.Contains(bucketName, "-.") { return errors.New("Bucket name contains invalid characters") } if strict { diff --git a/pkg/s3utils/utils_test.go b/pkg/s3utils/utils_test.go index 60f2a06ab..322b61932 100644 --- a/pkg/s3utils/utils_test.go +++ b/pkg/s3utils/utils_test.go @@ -335,6 +335,8 @@ func TestIsValidBucketName(t *testing.T) { {"my", errors.New("Bucket name cannot be smaller than 3 characters"), false}, {"", errors.New("Bucket name cannot be empty"), false}, {"my..bucket", errors.New("Bucket name contains invalid characters"), false}, + {"my.-bucket", errors.New("Bucket name contains invalid characters"), false}, + {"my-.bucket", errors.New("Bucket name contains invalid characters"), false}, {"192.168.1.168", errors.New("Bucket name cannot be an ip address"), false}, {":bucketname", errors.New("Bucket name contains invalid characters"), false}, {"_bucketName", errors.New("Bucket name contains invalid characters"), false}, @@ -381,6 +383,8 @@ func TestIsValidBucketNameStrict(t *testing.T) { {"my", errors.New("Bucket name cannot be smaller than 3 characters"), false}, {"", errors.New("Bucket name cannot be empty"), false}, {"my..bucket", errors.New("Bucket name contains invalid characters"), false}, + {"my.-bucket", errors.New("Bucket name contains invalid characters"), false}, + {"my-.bucket", errors.New("Bucket name contains invalid characters"), false}, {"192.168.1.168", errors.New("Bucket name cannot be an ip address"), false}, {"Mybucket", errors.New("Bucket name contains invalid characters"), false}, {"my.bucket.com", nil, true}, From 68eef499b4c530409962a5d1a12fa8c848587761 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Wed, 1 May 2019 22:26:17 +0000 Subject: [PATCH 020/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 6e329dabf..3fd7d9557 100644 --- a/api.go +++ b/api.go @@ -102,7 +102,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.24" + libraryVersion = "v6.0.25" ) // User Agent should always following the below style. From f33eae714a282af20fb59f4a3bf27fdb1d92bf6a Mon Sep 17 00:00:00 2001 From: Chris Johnston Date: Sat, 11 May 2019 03:04:25 -0400 Subject: [PATCH 021/215] support new ap-east-1 region (#1105) --- s3-endpoints.go | 1 + 1 file changed, 1 insertion(+) diff --git a/s3-endpoints.go b/s3-endpoints.go index ef4ae74b5..989f58c7e 100644 --- a/s3-endpoints.go +++ b/s3-endpoints.go @@ -29,6 +29,7 @@ var awsS3EndpointMap = map[string]string{ "eu-west-3": "s3.dualstack.eu-west-3.amazonaws.com", "eu-central-1": "s3.dualstack.eu-central-1.amazonaws.com", "eu-north-1": "s3.dualstack.eu-north-1.amazonaws.com", + "ap-east-1": "s3.dualstack.ap-east-1.amazonaws.com", "ap-south-1": "s3.dualstack.ap-south-1.amazonaws.com", "ap-southeast-1": "s3.dualstack.ap-southeast-1.amazonaws.com", "ap-southeast-2": "s3.dualstack.ap-southeast-2.amazonaws.com", From 18cecbac9714693401c118768c217df964a7ef9d Mon Sep 17 00:00:00 2001 From: Peter Szalai Date: Mon, 13 May 2019 23:28:42 +0100 Subject: [PATCH 022/215] [api] enable error tracing (#1091) --- api.go | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/api.go b/api.go index 3fd7d9557..9303c9b14 100644 --- a/api.go +++ b/api.go @@ -73,8 +73,9 @@ type Client struct { bucketLocCache *bucketLocationCache // Advanced functionality. - isTraceEnabled bool - traceOutput io.Writer + isTraceEnabled bool + traceErrorsOnly bool + traceOutput io.Writer // S3 specific accelerated endpoint. s3AccelerateEndpoint string @@ -373,10 +374,23 @@ func (c *Client) TraceOn(outputStream io.Writer) { c.isTraceEnabled = true } +// TraceErrorsOnlyOn - same as TraceOn, but only errors will be traced. +func (c *Client) TraceErrorsOnlyOn(outputStream io.Writer) { + c.TraceOn(outputStream) + c.traceErrorsOnly = true +} + +// TraceErrorsOnlyOff - Turns off the errors only tracing and everything will be traced after this call. +// If all tracing needs to be turned off, call TraceOff(). +func (c *Client) TraceErrorsOnlyOff() { + c.traceErrorsOnly = false +} + // TraceOff - disable HTTP tracing. func (c *Client) TraceOff() { // Disable tracing. c.isTraceEnabled = false + c.traceErrorsOnly = false } // SetS3TransferAccelerate - turns s3 accelerated endpoint on or off for all your @@ -516,8 +530,9 @@ func (c Client) do(req *http.Request) (*http.Response, error) { return nil, ErrInvalidArgument(msg) } - // If trace is enabled, dump http request and response. - if c.isTraceEnabled { + // If trace is enabled, dump http request and response, + // except when the traceErrorsOnly enabled and the response's status code is ok + if c.isTraceEnabled && !(c.traceErrorsOnly && resp.StatusCode == http.StatusOK) { err = c.dumpHTTP(req, resp) if err != nil { return nil, err From 1add5d55024584a5adf39da0e6584a4349aac221 Mon Sep 17 00:00:00 2001 From: xjewer Date: Wed, 15 May 2019 08:44:12 +0100 Subject: [PATCH 023/215] Retry on 504 Gateway timeout http status code. (#1104) --- retry.go | 1 + 1 file changed, 1 insertion(+) diff --git a/retry.go b/retry.go index bd16fbb5a..94176b0a1 100644 --- a/retry.go +++ b/retry.go @@ -143,6 +143,7 @@ var retryableHTTPStatusCodes = map[int]struct{}{ http.StatusInternalServerError: {}, http.StatusBadGateway: {}, http.StatusServiceUnavailable: {}, + http.StatusGatewayTimeout: {}, // Add more HTTP status codes here. } From c6c2912aa5522e5f5a505e6cba30e95f0d8456fa Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 23 May 2019 12:23:47 -0700 Subject: [PATCH 024/215] Allow domains with . at the end (#1110) Fixes #1109 --- pkg/s3utils/utils.go | 4 ++-- pkg/s3utils/utils_test.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/s3utils/utils.go b/pkg/s3utils/utils.go index 6de0a45aa..9af2997b7 100644 --- a/pkg/s3utils/utils.go +++ b/pkg/s3utils/utils.go @@ -47,8 +47,8 @@ func IsValidDomain(host string) bool { if host[len(host)-1:] == "_" || host[:1] == "_" { return false } - // host cannot start or end with a "." - if host[len(host)-1:] == "." || host[:1] == "." { + // host cannot start with a "." + if host[:1] == "." { return false } // All non alphanumeric characters are invalid. diff --git a/pkg/s3utils/utils_test.go b/pkg/s3utils/utils_test.go index 322b61932..587b5cd32 100644 --- a/pkg/s3utils/utils_test.go +++ b/pkg/s3utils/utils_test.go @@ -99,6 +99,7 @@ func TestIsValidDomain(t *testing.T) { {"s3.amz.test.com", true}, {"s3.%%", false}, {"localhost", true}, + {"localhost.", true}, // http://www.dns-sd.org/trailingdotsindomainnames.html {"-localhost", false}, {"", false}, {"\n \t", false}, From 28172b9253ed96432f877e2ca01d37941e3a2330 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Thu, 23 May 2019 22:26:54 +0000 Subject: [PATCH 025/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 9303c9b14..a4e0d77a8 100644 --- a/api.go +++ b/api.go @@ -103,7 +103,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.25" + libraryVersion = "v6.0.26" ) // User Agent should always following the below style. From da91b3bbdca31aef023d51f3ac74cec0e02b23ac Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 24 May 2019 15:35:15 -0700 Subject: [PATCH 026/215] Fix go module imports properly to honor semantic versioning (#1111) --- .travis.yml | 2 +- Makefile | 29 ++++++----- README.md | 4 +- README_zh_CN.md | 4 +- api-compose-object.go | 4 +- api-error-response.go | 2 +- api-get-lifecycle.go | 2 +- api-get-object-file.go | 2 +- api-get-object.go | 2 +- api-get-options.go | 2 +- api-get-policy.go | 2 +- api-list.go | 2 +- api-notification.go | 19 +++---- api-presigned.go | 4 +- api-put-bucket.go | 2 +- api-put-object-common.go | 4 +- api-put-object-copy.go | 2 +- api-put-object-file-context.go | 2 +- api-put-object-multipart.go | 4 +- api-put-object-streaming.go | 2 +- api-put-object.go | 4 +- api-remove.go | 2 +- api-select.go | 4 +- api-stat.go | 2 +- api.go | 14 ++--- api_unit_test.go | 23 ++------- appveyor.yml | 18 +++---- bucket-cache.go | 6 +-- bucket-cache_test.go | 12 ++--- bucket-notification.go | 2 +- core.go | 4 +- core_test.go | 2 - docs/API.md | 12 ++--- docs/checker.go.template | 2 +- docs/zh_CN/API.md | 20 ++++---- examples/minio/listenbucketnotification.go | 2 +- examples/s3/bucketexists.go | 2 +- examples/s3/composeobject.go | 15 +++--- examples/s3/copyobject.go | 2 +- examples/s3/fgetobject-context.go | 2 +- examples/s3/fgetobject.go | 2 +- examples/s3/fputencrypted-object.go | 5 +- examples/s3/fputobject-context.go | 2 +- examples/s3/fputobject.go | 2 +- examples/s3/get-encrypted-object.go | 4 +- examples/s3/getbucketlifecycle.go | 2 +- examples/s3/getbucketnotification.go | 2 +- examples/s3/getbucketpolicy.go | 2 +- examples/s3/getobject-client-encryption.go | 2 +- examples/s3/getobject-context.go | 2 +- examples/s3/getobject.go | 2 +- examples/s3/getobjectacl.go | 2 +- examples/s3/listbuckets.go | 2 +- examples/s3/listincompleteuploads.go | 2 +- examples/s3/listobjects-N.go | 2 +- examples/s3/listobjects.go | 2 +- examples/s3/listobjectsV2.go | 2 +- examples/s3/makebucket.go | 2 +- examples/s3/presignedgetobject.go | 2 +- examples/s3/presignedheadobject.go | 2 +- examples/s3/presignedpostpolicy.go | 2 +- examples/s3/presignedputobject.go | 2 +- examples/s3/put-encrypted-object.go | 4 +- examples/s3/putobject-client-encryption.go | 2 +- examples/s3/putobject-context.go | 2 +- examples/s3/putobject-getobject-sse.go | 5 +- examples/s3/putobject-progress.go | 2 +- examples/s3/putobject-s3-accelerate.go | 2 +- examples/s3/putobject-streaming.go | 2 +- examples/s3/putobject.go | 2 +- examples/s3/removeallbucketnotification.go | 2 +- examples/s3/removebucket.go | 2 +- examples/s3/removeincompleteupload.go | 2 +- examples/s3/removeobject.go | 2 +- examples/s3/removeobjects.go | 2 +- examples/s3/selectobject.go | 2 +- examples/s3/setbucketlifecycle.go | 2 +- examples/s3/setbucketnotification.go | 2 +- examples/s3/setbucketpolicy.go | 2 +- examples/s3/statobject.go | 2 +- functional_tests.go | 4 +- go.mod | 22 ++++---- go.sum | 60 ++++++++++++++++------ pkg/credentials/iam_aws_test.go | 4 +- pkg/policy/bucket-policy-condition.go | 2 +- pkg/policy/bucket-policy-condition_test.go | 2 +- pkg/policy/bucket-policy.go | 2 +- pkg/policy/bucket-policy_test.go | 2 +- pkg/s3signer/request-signature-v2.go | 2 +- pkg/s3signer/request-signature-v4.go | 2 +- pkg/set/stringset_test.go | 2 +- staticcheck.conf | 1 + utils.go | 4 +- utils_test.go | 2 +- 94 files changed, 224 insertions(+), 223 deletions(-) create mode 100644 staticcheck.conf diff --git a/.travis.yml b/.travis.yml index 7ed7df14e..7adb883ff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ env: - ARCH=i686 go: -- 1.11.x +- 1.12.x - tip matrix: diff --git a/Makefile b/Makefile index 51c8ca266..fa0936c0f 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,20 @@ all: checks -checks: - @go get -t ./... - @go vet ./... - @SERVER_ENDPOINT=play.min.io:9000 ACCESS_KEY=Q3AM3UQ867SPQQA43P2F SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG ENABLE_HTTPS=1 MINT_MODE=full go test -race -v ./... - @go get github.com/dustin/go-humanize/... - @go get github.com/sirupsen/logrus/... - @SERVER_ENDPOINT=play.min.io:9000 ACCESS_KEY=Q3AM3UQ867SPQQA43P2F SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG ENABLE_HTTPS=1 MINT_MODE=full go run functional_tests.go +.PHONY: examples docs + +checks: vet test examples docs functional-test + +vet: + @GO111MODULE=on go vet ./... + +test: + @GO111MODULE=on SERVER_ENDPOINT=play.min.io:9000 ACCESS_KEY=Q3AM3UQ867SPQQA43P2F SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG ENABLE_HTTPS=1 MINT_MODE=full go test -race -v ./... + +examples: @mkdir -p /tmp/examples && for i in $(echo examples/s3/*); do go build -o /tmp/examples/$(basename ${i:0:-3}) ${i}; done - @go get -u github.com/a8m/mark/... - @go get -u github.com/minio/cli/... - @go get -u golang.org/x/tools/cmd/goimports - @go get -u github.com/gernest/wow/... - @go build docs/validator.go && ./validator -m docs/API.md -t docs/checker.go.tpl + +docs: + @(cd docs; GO111MODULE=on go build validator.go && ./validator -m ../docs/API.md -t checker.go.tpl) + +functional-test: + @GO111MODULE=on SERVER_ENDPOINT=play.min.io:9000 ACCESS_KEY=Q3AM3UQ867SPQQA43P2F SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG ENABLE_HTTPS=1 MINT_MODE=full go run functional_tests.go diff --git a/README.md b/README.md index 5464390fa..8a66bf5c9 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ MinIO client requires the following four parameters specified to connect to an A package main import ( - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" "log" ) @@ -56,7 +56,7 @@ We will use the MinIO server running at [https://play.min.io:9000](https://play. package main import ( - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" "log" ) diff --git a/README_zh_CN.md b/README_zh_CN.md index 061f41743..c9c5e3133 100644 --- a/README_zh_CN.md +++ b/README_zh_CN.md @@ -38,7 +38,7 @@ MinIO client需要以下4个参数来连接与Amazon S3兼容的对象存储。 package main import ( - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" "log" ) @@ -68,7 +68,7 @@ func main() { package main import ( - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" "log" ) diff --git a/api-compose-object.go b/api-compose-object.go index b4dddacb2..748b558ce 100644 --- a/api-compose-object.go +++ b/api-compose-object.go @@ -28,8 +28,8 @@ import ( "strings" "time" - "github.com/minio/minio-go/pkg/encrypt" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/encrypt" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // DestinationInfo - type with information about the object to be diff --git a/api-error-response.go b/api-error-response.go index 5dc8a0a90..726fdd274 100644 --- a/api-error-response.go +++ b/api-error-response.go @@ -60,7 +60,7 @@ type ErrorResponse struct { // // For example: // -// import s3 "github.com/minio/minio-go" +// import s3 "github.com/minio/minio-go/v6" // ... // ... // reader, stat, err := s3.GetObject(...) diff --git a/api-get-lifecycle.go b/api-get-lifecycle.go index 96ccfe902..a24d03e37 100644 --- a/api-get-lifecycle.go +++ b/api-get-lifecycle.go @@ -23,7 +23,7 @@ import ( "net/http" "net/url" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // GetBucketLifecycle - get bucket lifecycle. diff --git a/api-get-object-file.go b/api-get-object-file.go index 98837bfa3..9c82a7c4f 100644 --- a/api-get-object-file.go +++ b/api-get-object-file.go @@ -23,7 +23,7 @@ import ( "os" "path/filepath" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // FGetObjectWithContext - download contents of an object to a local file. diff --git a/api-get-object.go b/api-get-object.go index a00ddaa89..e4cd851f4 100644 --- a/api-get-object.go +++ b/api-get-object.go @@ -27,7 +27,7 @@ import ( "sync" "time" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // GetObject - returns an seekable, readable object. diff --git a/api-get-options.go b/api-get-options.go index 323b1c909..538fd1a05 100644 --- a/api-get-options.go +++ b/api-get-options.go @@ -22,7 +22,7 @@ import ( "net/http" "time" - "github.com/minio/minio-go/pkg/encrypt" + "github.com/minio/minio-go/v6/pkg/encrypt" ) // GetObjectOptions are used to specify additional headers or options diff --git a/api-get-policy.go b/api-get-policy.go index 035685d04..bc1d10530 100644 --- a/api-get-policy.go +++ b/api-get-policy.go @@ -23,7 +23,7 @@ import ( "net/http" "net/url" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // GetBucketPolicy - get bucket policy at a given path. diff --git a/api-list.go b/api-list.go index 6c4259eb4..b9c0f5d8d 100644 --- a/api-list.go +++ b/api-list.go @@ -25,7 +25,7 @@ import ( "net/url" "strings" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // ListBuckets list all buckets owned by this authenticated user. diff --git a/api-notification.go b/api-notification.go index 38a3f992f..a8d9fc613 100644 --- a/api-notification.go +++ b/api-notification.go @@ -21,12 +21,11 @@ import ( "bufio" "context" "encoding/json" - "io" "net/http" "net/url" "time" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // GetBucketNotification - get bucket notification at a given path. @@ -196,30 +195,24 @@ func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, even // Initialize a new bufio scanner, to read line by line. bio := bufio.NewScanner(resp.Body) - // Close the response body. - defer resp.Body.Close() - // Unmarshal each line, returns marshalled values. for bio.Scan() { var notificationInfo NotificationInfo if err = json.Unmarshal(bio.Bytes(), ¬ificationInfo); err != nil { + closeResponse(resp) continue } // Send notificationInfo select { case notificationInfoCh <- notificationInfo: case <-doneCh: + closeResponse(resp) return } } - // Look for any underlying errors. - if err = bio.Err(); err != nil { - // For an unexpected connection drop from server, we close the body - // and re-connect. - if err == io.ErrUnexpectedEOF { - resp.Body.Close() - } - } + + // Close current connection before looping further. + closeResponse(resp) } }(notificationInfoCh) diff --git a/api-presigned.go b/api-presigned.go index 8ffcdd712..e2d68b0ec 100644 --- a/api-presigned.go +++ b/api-presigned.go @@ -23,8 +23,8 @@ import ( "net/url" "time" - "github.com/minio/minio-go/pkg/s3signer" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3signer" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // presignURL - Returns a presigned URL for an input 'method'. diff --git a/api-put-bucket.go b/api-put-bucket.go index 69d33e315..28613a229 100644 --- a/api-put-bucket.go +++ b/api-put-bucket.go @@ -26,7 +26,7 @@ import ( "net/url" "strings" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) /// Bucket operations diff --git a/api-put-object-common.go b/api-put-object-common.go index 00d98b290..0312fd317 100644 --- a/api-put-object-common.go +++ b/api-put-object-common.go @@ -23,7 +23,7 @@ import ( "math" "os" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // Verify if reader is *minio.Object @@ -105,7 +105,7 @@ func optimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCou configuredPartSize = minPartSize // Use floats for part size for all calculations to avoid // overflows during float64 to int64 conversions. - partSizeFlt = math.Ceil(float64(objectSize / maxPartsCount)) + partSizeFlt = float64(objectSize / maxPartsCount) partSizeFlt = math.Ceil(partSizeFlt/float64(configuredPartSize)) * float64(configuredPartSize) } diff --git a/api-put-object-copy.go b/api-put-object-copy.go index c6ecf39d8..19e58add8 100644 --- a/api-put-object-copy.go +++ b/api-put-object-copy.go @@ -23,7 +23,7 @@ import ( "io/ioutil" "net/http" - "github.com/minio/minio-go/pkg/encrypt" + "github.com/minio/minio-go/v6/pkg/encrypt" ) // CopyObject - copy a source object into a new object diff --git a/api-put-object-file-context.go b/api-put-object-file-context.go index 3574a6b5b..fb22c0d64 100644 --- a/api-put-object-file-context.go +++ b/api-put-object-file-context.go @@ -23,7 +23,7 @@ import ( "os" "path/filepath" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // FPutObjectWithContext - Create an object in a bucket, with contents from file at filePath. Allows request cancellation. diff --git a/api-put-object-multipart.go b/api-put-object-multipart.go index 95a2599b0..ab284f986 100644 --- a/api-put-object-multipart.go +++ b/api-put-object-multipart.go @@ -33,8 +33,8 @@ import ( "strconv" "strings" - "github.com/minio/minio-go/pkg/encrypt" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/encrypt" + "github.com/minio/minio-go/v6/pkg/s3utils" ) func (c Client) putObjectMultipart(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, diff --git a/api-put-object-streaming.go b/api-put-object-streaming.go index 0035def3e..8bea9b4b8 100644 --- a/api-put-object-streaming.go +++ b/api-put-object-streaming.go @@ -25,7 +25,7 @@ import ( "sort" "strings" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // putObjectMultipartStream - upload a large object using diff --git a/api-put-object.go b/api-put-object.go index 4ea9c495a..0ae7e4c69 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -26,8 +26,8 @@ import ( "runtime/debug" "sort" - "github.com/minio/minio-go/pkg/encrypt" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/encrypt" + "github.com/minio/minio-go/v6/pkg/s3utils" "golang.org/x/net/http/httpguts" ) diff --git a/api-remove.go b/api-remove.go index 29f6f347e..e919be17d 100644 --- a/api-remove.go +++ b/api-remove.go @@ -25,7 +25,7 @@ import ( "net/http" "net/url" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // RemoveBucket deletes the bucket name. diff --git a/api-select.go b/api-select.go index 0b5762450..b5ce13112 100644 --- a/api-select.go +++ b/api-select.go @@ -31,8 +31,8 @@ import ( "net/url" "strings" - "github.com/minio/minio-go/pkg/encrypt" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/encrypt" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // CSVFileHeaderInfo - is the parameter for whether to utilize headers. diff --git a/api-stat.go b/api-stat.go index 15d0af39e..c6fb47d63 100644 --- a/api-stat.go +++ b/api-stat.go @@ -24,7 +24,7 @@ import ( "strings" "time" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // BucketExists verify if bucket exists and you have permission to access it. diff --git a/api.go b/api.go index a4e0d77a8..0c358f368 100644 --- a/api.go +++ b/api.go @@ -41,9 +41,9 @@ import ( "golang.org/x/net/publicsuffix" - "github.com/minio/minio-go/pkg/credentials" - "github.com/minio/minio-go/pkg/s3signer" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/credentials" + "github.com/minio/minio-go/v6/pkg/s3signer" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // Client implements Amazon S3 compatible methods. @@ -264,7 +264,7 @@ func (c *Client) redirectHeaders(req *http.Request, via []*http.Request) error { case signerType.IsV2(): return errors.New("signature V2 cannot support redirection") case signerType.IsV4(): - req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, getDefaultLocation(*c.endpointURL, region)) + s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, getDefaultLocation(*c.endpointURL, region)) } } return nil @@ -331,10 +331,6 @@ func privateNew(endpoint string, creds *credentials.Credentials, secure bool, re func (c *Client) SetAppInfo(appName string, appVersion string) { // if app name and version not set, we do not set a new user agent. if appName != "" && appVersion != "" { - c.appInfo = struct { - appName string - appVersion string - }{} c.appInfo.appName = appName c.appInfo.appVersion = appVersion } @@ -661,7 +657,7 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque if errResponse.Code == "AuthorizationHeaderMalformed" || errResponse.Code == "InvalidRegion" { if metadata.bucketName != "" && errResponse.Region != "" { // Gather Cached location only if bucketName is present. - if _, cachedLocationError := c.bucketLocCache.Get(metadata.bucketName); cachedLocationError != false { + if _, cachedOk := c.bucketLocCache.Get(metadata.bucketName); cachedOk { c.bucketLocCache.Set(metadata.bucketName, errResponse.Region) continue // Retry. } diff --git a/api_unit_test.go b/api_unit_test.go index 48195325b..c01578052 100644 --- a/api_unit_test.go +++ b/api_unit_test.go @@ -18,24 +18,13 @@ package minio import ( - "net/http" "net/url" "testing" - "github.com/minio/minio-go/pkg/credentials" - "github.com/minio/minio-go/pkg/policy" + "github.com/minio/minio-go/v6/pkg/credentials" + "github.com/minio/minio-go/v6/pkg/policy" ) -type customReader struct{} - -func (c *customReader) Read(p []byte) (n int, err error) { - return 0, nil -} - -func (c *customReader) Size() (n int64) { - return 10 -} - // Tests valid hosts for location. func TestValidBucketLocation(t *testing.T) { s3Hosts := []struct { @@ -65,14 +54,8 @@ func TestErrorResponse(t *testing.T) { t.Fatal("Type conversion failed, we have an empty struct.") } - // Test http response decoding. - var httpResponse *http.Response - // Set empty variables - httpResponse = nil - var bucketName, objectName string - // Should fail with invalid argument. - err = httpRespToErrorResponse(httpResponse, bucketName, objectName) + err = httpRespToErrorResponse(nil, "", "") errResp = ToErrorResponse(err) if errResp.Code != "InvalidArgument" { t.Fatal("Empty response input should return invalid argument.") diff --git a/appveyor.yml b/appveyor.yml index 48ea6e77d..39a33d876 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -9,28 +9,24 @@ clone_folder: c:\gopath\src\github.com\minio\minio-go # environment variables environment: GOPATH: c:\gopath - GO15VENDOREXPERIMENT: 1 + GO111MODULE: on # scripts that run after cloning repository install: - set PATH=%GOPATH%\bin;c:\go\bin;%PATH% - go version - go env - - go get -u golang.org/x/lint/golint - - go get -u github.com/remyoudompheng/go-misc/deadcode - - go get -u github.com/gordonklaus/ineffassign - - go get -u golang.org/x/crypto/argon2 - - go get -t ./... + - go get golang.org/x/lint/golint + - go get honnef.co/go/tools/cmd/staticcheck # to run your custom scripts instead of automatic MSBuild build_script: - go vet ./... - gofmt -s -l . - - golint -set_exit_status github.com/minio/minio-go... - - deadcode - - ineffassign . - - go test -short -v - - go test -short -race -v + - golint -set_exit_status github.com/minio/minio-go/... + - staticcheck + - go test -short -v ./... + - go test -short -race -v ./... # to disable automatic tests test: off diff --git a/bucket-cache.go b/bucket-cache.go index 4ea28e569..53a8a9c7f 100644 --- a/bucket-cache.go +++ b/bucket-cache.go @@ -24,9 +24,9 @@ import ( "path" "sync" - "github.com/minio/minio-go/pkg/credentials" - "github.com/minio/minio-go/pkg/s3signer" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/credentials" + "github.com/minio/minio-go/v6/pkg/s3signer" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // bucketLocationCache - Provides simple mechanism to hold bucket diff --git a/bucket-cache_test.go b/bucket-cache_test.go index 477468ea2..8e4fe8fff 100644 --- a/bucket-cache_test.go +++ b/bucket-cache_test.go @@ -27,8 +27,8 @@ import ( "reflect" "testing" - "github.com/minio/minio-go/pkg/credentials" - "github.com/minio/minio-go/pkg/s3signer" + "github.com/minio/minio-go/v6/pkg/credentials" + "github.com/minio/minio-go/v6/pkg/s3signer" ) // Test validates `newBucketLocationCache`. @@ -67,7 +67,7 @@ func TestBucketLocationCacheOps(t *testing.T) { func TestGetBucketLocationRequest(t *testing.T) { // Generates expected http request for getBucketLocation. // Used for asserting with the actual request generated. - createExpectedRequest := func(c *Client, bucketName string, req *http.Request) (*http.Request, error) { + createExpectedRequest := func(c *Client, bucketName string) (*http.Request, error) { // Set location query. urlValues := make(url.Values) urlValues.Set("location", "") @@ -78,8 +78,7 @@ func TestGetBucketLocationRequest(t *testing.T) { targetURL.RawQuery = urlValues.Encode() // Get a new HTTP request for the method. - var err error - req, err = http.NewRequest("GET", targetURL.String(), nil) + req, err := http.NewRequest("GET", targetURL.String(), nil) if err != nil { return nil, err } @@ -251,8 +250,7 @@ func TestGetBucketLocationRequest(t *testing.T) { // Test passes as expected, but the output values are verified for correctness here. if err == nil && testCase.shouldPass { - expectedReq := &http.Request{} - expectedReq, err = createExpectedRequest(client, testCase.bucketName, expectedReq) + expectedReq, err := createExpectedRequest(client, testCase.bucketName) if err != nil { t.Fatalf("Test %d: Expected request Creation failed", i+1) } diff --git a/bucket-notification.go b/bucket-notification.go index 11fc299ff..4714eadad 100644 --- a/bucket-notification.go +++ b/bucket-notification.go @@ -20,7 +20,7 @@ package minio import ( "encoding/xml" - "github.com/minio/minio-go/pkg/set" + "github.com/minio/minio-go/v6/pkg/set" ) // NotificationEventType is a S3 notification event associated to the bucket notification configuration diff --git a/core.go b/core.go index afd4139ee..ba574c44e 100644 --- a/core.go +++ b/core.go @@ -22,7 +22,7 @@ import ( "io" "strings" - "github.com/minio/minio-go/pkg/encrypt" + "github.com/minio/minio-go/v6/pkg/encrypt" ) // Core - Inherits Client and adds new methods to expose the low level S3 APIs. @@ -84,7 +84,7 @@ func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Ba opts.ContentType = v } else if strings.ToLower(k) == "cache-control" { opts.CacheControl = v - } else if strings.ToLower(k) == strings.ToLower(amzWebsiteRedirectLocation) { + } else if strings.EqualFold(k, amzWebsiteRedirectLocation) { opts.WebsiteRedirectLocation = v } else { m[k] = metadata[k] diff --git a/core_test.go b/core_test.go index 525d6f6d7..b171bf822 100644 --- a/core_test.go +++ b/core_test.go @@ -35,8 +35,6 @@ const ( enableSecurity = "ENABLE_HTTPS" ) -// Minimum part size -const MinPartSize = 1024 * 1024 * 64 const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" const ( letterIdxBits = 6 // 6 bits to represent a letter index diff --git a/docs/API.md b/docs/API.md index b868443e8..7f580cfa5 100644 --- a/docs/API.md +++ b/docs/API.md @@ -10,7 +10,7 @@ package main import ( "fmt" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { @@ -34,7 +34,7 @@ package main import ( "fmt" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { @@ -393,7 +393,7 @@ __minio.GetObjectOptions__ |Field | Type | Description | |:---|:---|:---| -| `opts.ServerSideEncryption` | _encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go) | +| `opts.ServerSideEncryption` | _encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v6) | __Return Value__ @@ -552,7 +552,7 @@ __minio.PutObjectOptions__ | `opts.ContentDisposition` | _string_ | Content disposition of object, "inline" | | `opts.ContentLanguage` | _string_ | Content language of object, e.g "French" | | `opts.CacheControl` | _string_ | Used to specify directives for caching mechanisms in both requests and responses e.g "max-age=600"| -| `opts.ServerSideEncryption` | _encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go) | +| `opts.ServerSideEncryption` | _encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v6) | | `opts.StorageClass` | _string_ | Specify storage class for the object. Supported values for MinIO server are `REDUCED_REDUNDANCY` and `STANDARD` | | `opts.WebsiteRedirectLocation` | _string_ | Specify a redirect for the object, to another object in the same bucket or to a external URL. | @@ -771,7 +771,7 @@ __Parameters__ | :--- | :--- | :--- | | `bucket` | _string_ | Name of the source bucket | | `object` | _string_ | Name of the source object | -| `sse` | _*encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go) | +| `sse` | _*encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v6) | __Example__ @@ -824,7 +824,7 @@ __Parameters__ | :--- | :--- | :--- | | `bucket` | _string_ | Name of the destination bucket | | `object` | _string_ | Name of the destination object | -| `sse` | _*encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go) | | +| `sse` | _*encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v6) | | | `userMeta` | _map[string]string_ | User metadata to be set on the destination. If nil, with only one source, user-metadata is copied from source. | __Example__ diff --git a/docs/checker.go.template b/docs/checker.go.template index 4e332da42..750e1ec39 100644 --- a/docs/checker.go.template +++ b/docs/checker.go.template @@ -3,7 +3,7 @@ package main import ( "fmt" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/docs/zh_CN/API.md b/docs/zh_CN/API.md index 583892f6a..f6d84032f 100644 --- a/docs/zh_CN/API.md +++ b/docs/zh_CN/API.md @@ -10,7 +10,7 @@ package main import ( "fmt" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { @@ -34,7 +34,7 @@ package main import ( "fmt" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { @@ -379,7 +379,7 @@ __minio.GetObjectOptions__ |参数 | 类型 | 描述 | |:---|:---|:---| -| `opts.Materials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口,(更多信息,请看https://godoc.org/github.com/minio/minio-go) | +| `opts.Materials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口,(更多信息,请看https://godoc.org/github.com/minio/minio-go/v6) | __返回值__ @@ -524,7 +524,7 @@ __参数__ |`bucketName` | _string_ |存储桶名称 | |`objectName` | _string_ |对象的名称 | |`filePath` | _string_ |下载后保存的路径| -|`materials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口,(更多信息,请看https://godoc.org/github.com/minio/minio-go) | +|`materials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口,(更多信息,请看https://godoc.org/github.com/minio/minio-go/v6) | __示例__ @@ -573,7 +573,7 @@ __minio.PutObjectOptions__ | `opts.ContentEncoding` | _string_ | 对象的Content encoding,例如"gzip" | | `opts.ContentDisposition` | _string_ | 对象的Content disposition, "inline" | | `opts.CacheControl` | _string_ | 指定针对请求和响应的缓存机制,例如"max-age=600"| -| `opts.EncryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口,(更多信息,请看https://godoc.org/github.com/minio/minio-go) | +| `opts.EncryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口,(更多信息,请看https://godoc.org/github.com/minio/minio-go/v6) | __示例__ @@ -1176,7 +1176,7 @@ __参数__ |:---|:---| :---| |`bucketName` | _string_ | 存储桶名称 | |`objectName` | _string_ | 对象的名称 | -|`encryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口,(更多信息,请看https://godoc.org/github.com/minio/minio-go) | +|`encryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口,(更多信息,请看https://godoc.org/github.com/minio/minio-go/v6) | __返回值__ @@ -1233,7 +1233,7 @@ __参数__ |`bucketName` | _string_ |存储桶名称 | |`objectName` | _string_ |对象的名称 | |`reader` | _io.Reader_ |任何实现io.Reader的Go类型 | -|`encryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口,(更多信息,请看https://godoc.org/github.com/minio/minio-go) | +|`encryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口,(更多信息,请看https://godoc.org/github.com/minio/minio-go/v6) | __示例__ @@ -1295,7 +1295,7 @@ __参数__ |`bucketName` | _string_ |存储桶名称 | |`objectName` | _string_ |对象的名称 | |`filePath` | _string_ |要上传的文件的路径 | -|`encryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口,(更多信息,请看https://godoc.org/github.com/minio/minio-go) | +|`encryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口,(更多信息,请看https://godoc.org/github.com/minio/minio-go/v6) | __示例__ @@ -1489,7 +1489,7 @@ fmt.Printf("%s\n", url) ### SetBucketPolicy(bucketname, objectPrefix string, policy policy.BucketPolicy) error 给存储桶或者对象前缀设置访问权限。 -必须引入`github.com/minio/minio-go/pkg/policy`包。 +必须引入`github.com/minio/minio-go/v6/pkg/policy`包。 __参数__ @@ -1530,7 +1530,7 @@ if err != nil { ### GetBucketPolicy(bucketName, objectPrefix string) (policy.BucketPolicy, error) 获取存储桶或者对象前缀的访问权限。 -必须引入`github.com/minio/minio-go/pkg/policy`包。 +必须引入`github.com/minio/minio-go/v6/pkg/policy`包。 __参数__ diff --git a/examples/minio/listenbucketnotification.go b/examples/minio/listenbucketnotification.go index eff0219a6..3082f9f21 100644 --- a/examples/minio/listenbucketnotification.go +++ b/examples/minio/listenbucketnotification.go @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/bucketexists.go b/examples/s3/bucketexists.go index 84a7595f5..e4b0a3a0e 100644 --- a/examples/s3/bucketexists.go +++ b/examples/s3/bucketexists.go @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/composeobject.go b/examples/s3/composeobject.go index 1966383ce..78bfe9500 100644 --- a/examples/s3/composeobject.go +++ b/examples/s3/composeobject.go @@ -22,7 +22,8 @@ package main import ( "log" - minio "github.com/minio/minio-go" + minio "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v6/pkg/encrypt" ) func main() { @@ -44,27 +45,27 @@ func main() { // Prepare source decryption key (here we assume same key to // decrypt all source objects.) - decKey := minio.NewSSEInfo([]byte{1, 2, 3}, "") + decKey, _ := encrypt.NewSSEC([]byte{1, 2, 3}) // Source objects to concatenate. We also specify decryption // key for each - src1 := minio.NewSourceInfo("bucket1", "object1", &decKey) + src1 := minio.NewSourceInfo("bucket1", "object1", decKey) src1.SetMatchETagCond("31624deb84149d2f8ef9c385918b653a") - src2 := minio.NewSourceInfo("bucket2", "object2", &decKey) + src2 := minio.NewSourceInfo("bucket2", "object2", decKey) src2.SetMatchETagCond("f8ef9c385918b653a31624deb84149d2") - src3 := minio.NewSourceInfo("bucket3", "object3", &decKey) + src3 := minio.NewSourceInfo("bucket3", "object3", decKey) src3.SetMatchETagCond("5918b653a31624deb84149d2f8ef9c38") // Create slice of sources. srcs := []minio.SourceInfo{src1, src2, src3} // Prepare destination encryption key - encKey := minio.NewSSEInfo([]byte{8, 9, 0}, "") + encKey, _ := encrypt.NewSSEC([]byte{8, 9, 0}) // Create destination info - dst, err := minio.NewDestinationInfo("bucket", "object", &encKey, nil) + dst, err := minio.NewDestinationInfo("bucket", "object", encKey, nil) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/copyobject.go b/examples/s3/copyobject.go index 9dcfab168..1c3f65a3a 100644 --- a/examples/s3/copyobject.go +++ b/examples/s3/copyobject.go @@ -23,7 +23,7 @@ import ( "log" "time" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/fgetobject-context.go b/examples/s3/fgetobject-context.go index 13acc3ce7..f3dc74d7b 100644 --- a/examples/s3/fgetobject-context.go +++ b/examples/s3/fgetobject-context.go @@ -25,7 +25,7 @@ import ( "context" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/fgetobject.go b/examples/s3/fgetobject.go index b747a907f..76dd59f3f 100644 --- a/examples/s3/fgetobject.go +++ b/examples/s3/fgetobject.go @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/fputencrypted-object.go b/examples/s3/fputencrypted-object.go index b0dc24006..95eefeafd 100644 --- a/examples/s3/fputencrypted-object.go +++ b/examples/s3/fputencrypted-object.go @@ -22,9 +22,8 @@ package main import ( "log" - "github.com/minio/minio-go/pkg/encrypt" - - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v6/pkg/encrypt" ) func main() { diff --git a/examples/s3/fputobject-context.go b/examples/s3/fputobject-context.go index 1cd030c5a..0aa300c2d 100644 --- a/examples/s3/fputobject-context.go +++ b/examples/s3/fputobject-context.go @@ -25,7 +25,7 @@ import ( "context" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/fputobject.go b/examples/s3/fputobject.go index 6ce0de68f..2fc359545 100644 --- a/examples/s3/fputobject.go +++ b/examples/s3/fputobject.go @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/get-encrypted-object.go b/examples/s3/get-encrypted-object.go index 43be44e6b..260770daf 100644 --- a/examples/s3/get-encrypted-object.go +++ b/examples/s3/get-encrypted-object.go @@ -24,8 +24,8 @@ import ( "log" "os" - "github.com/minio/minio-go" - "github.com/minio/minio-go/pkg/encrypt" + "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v6/pkg/encrypt" ) func main() { diff --git a/examples/s3/getbucketlifecycle.go b/examples/s3/getbucketlifecycle.go index 3086ae7b7..1982a7639 100644 --- a/examples/s3/getbucketlifecycle.go +++ b/examples/s3/getbucketlifecycle.go @@ -25,7 +25,7 @@ import ( "os" "strings" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/getbucketnotification.go b/examples/s3/getbucketnotification.go index 9210dc5a0..1d711fc68 100644 --- a/examples/s3/getbucketnotification.go +++ b/examples/s3/getbucketnotification.go @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/getbucketpolicy.go b/examples/s3/getbucketpolicy.go index e8646f874..b10e933ac 100644 --- a/examples/s3/getbucketpolicy.go +++ b/examples/s3/getbucketpolicy.go @@ -25,7 +25,7 @@ import ( "os" "strings" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/getobject-client-encryption.go b/examples/s3/getobject-client-encryption.go index 6b06dca81..33fb0d2ed 100644 --- a/examples/s3/getobject-client-encryption.go +++ b/examples/s3/getobject-client-encryption.go @@ -24,7 +24,7 @@ import ( "os" "path" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" "github.com/minio/sio" "golang.org/x/crypto/argon2" ) diff --git a/examples/s3/getobject-context.go b/examples/s3/getobject-context.go index 4e68daa5b..1764d8aed 100644 --- a/examples/s3/getobject-context.go +++ b/examples/s3/getobject-context.go @@ -27,7 +27,7 @@ import ( "context" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/getobject.go b/examples/s3/getobject.go index e09401aa7..eab9fc5aa 100644 --- a/examples/s3/getobject.go +++ b/examples/s3/getobject.go @@ -24,7 +24,7 @@ import ( "log" "os" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/getobjectacl.go b/examples/s3/getobjectacl.go index b397fa45f..f557009b2 100644 --- a/examples/s3/getobjectacl.go +++ b/examples/s3/getobjectacl.go @@ -23,7 +23,7 @@ import ( "fmt" "log" - minio "github.com/minio/minio-go" + minio "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/listbuckets.go b/examples/s3/listbuckets.go index fa1861d54..9b1e23e97 100644 --- a/examples/s3/listbuckets.go +++ b/examples/s3/listbuckets.go @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/listincompleteuploads.go b/examples/s3/listincompleteuploads.go index 1fe4b2ac9..da9bf59a7 100644 --- a/examples/s3/listincompleteuploads.go +++ b/examples/s3/listincompleteuploads.go @@ -23,7 +23,7 @@ import ( "fmt" "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/listobjects-N.go b/examples/s3/listobjects-N.go index 86e6df599..12afbbd42 100644 --- a/examples/s3/listobjects-N.go +++ b/examples/s3/listobjects-N.go @@ -22,7 +22,7 @@ package main import ( "fmt" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/listobjects.go b/examples/s3/listobjects.go index 5c9d0e94b..34fed8445 100644 --- a/examples/s3/listobjects.go +++ b/examples/s3/listobjects.go @@ -22,7 +22,7 @@ package main import ( "fmt" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/listobjectsV2.go b/examples/s3/listobjectsV2.go index 2750a82a9..7443e8b1a 100644 --- a/examples/s3/listobjectsV2.go +++ b/examples/s3/listobjectsV2.go @@ -22,7 +22,7 @@ package main import ( "fmt" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/makebucket.go b/examples/s3/makebucket.go index fe7a9a5c6..1e179957f 100644 --- a/examples/s3/makebucket.go +++ b/examples/s3/makebucket.go @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/presignedgetobject.go b/examples/s3/presignedgetobject.go index 0a4c797ea..dc870a029 100644 --- a/examples/s3/presignedgetobject.go +++ b/examples/s3/presignedgetobject.go @@ -24,7 +24,7 @@ import ( "net/url" "time" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/presignedheadobject.go b/examples/s3/presignedheadobject.go index d9354550b..8cec788c9 100644 --- a/examples/s3/presignedheadobject.go +++ b/examples/s3/presignedheadobject.go @@ -24,7 +24,7 @@ import ( "net/url" "time" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/presignedpostpolicy.go b/examples/s3/presignedpostpolicy.go index af6f6f106..25e0f74d7 100644 --- a/examples/s3/presignedpostpolicy.go +++ b/examples/s3/presignedpostpolicy.go @@ -24,7 +24,7 @@ import ( "log" "time" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/presignedputobject.go b/examples/s3/presignedputobject.go index 516a3b01e..d200faa5b 100644 --- a/examples/s3/presignedputobject.go +++ b/examples/s3/presignedputobject.go @@ -23,7 +23,7 @@ import ( "log" "time" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/put-encrypted-object.go b/examples/s3/put-encrypted-object.go index d15d0190b..7d4cde286 100644 --- a/examples/s3/put-encrypted-object.go +++ b/examples/s3/put-encrypted-object.go @@ -23,8 +23,8 @@ import ( "log" "os" - "github.com/minio/minio-go" - "github.com/minio/minio-go/pkg/encrypt" + "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v6/pkg/encrypt" ) func main() { diff --git a/examples/s3/putobject-client-encryption.go b/examples/s3/putobject-client-encryption.go index b46b3476f..0c0d7bb2a 100644 --- a/examples/s3/putobject-client-encryption.go +++ b/examples/s3/putobject-client-encryption.go @@ -24,7 +24,7 @@ import ( "os" "path" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" "github.com/minio/sio" "golang.org/x/crypto/argon2" ) diff --git a/examples/s3/putobject-context.go b/examples/s3/putobject-context.go index 1c40372d6..6a476d49e 100644 --- a/examples/s3/putobject-context.go +++ b/examples/s3/putobject-context.go @@ -26,7 +26,7 @@ import ( "context" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/putobject-getobject-sse.go b/examples/s3/putobject-getobject-sse.go index 1415d6137..110c323a2 100644 --- a/examples/s3/putobject-getobject-sse.go +++ b/examples/s3/putobject-getobject-sse.go @@ -24,9 +24,8 @@ import ( "io/ioutil" "log" - "github.com/minio/minio-go/pkg/encrypt" - - minio "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v6/pkg/encrypt" ) func main() { diff --git a/examples/s3/putobject-progress.go b/examples/s3/putobject-progress.go index fc034c4b0..54ce7cbfb 100644 --- a/examples/s3/putobject-progress.go +++ b/examples/s3/putobject-progress.go @@ -23,7 +23,7 @@ import ( "log" "github.com/cheggaaa/pb" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/putobject-s3-accelerate.go b/examples/s3/putobject-s3-accelerate.go index 5203049c9..d36af0a06 100644 --- a/examples/s3/putobject-s3-accelerate.go +++ b/examples/s3/putobject-s3-accelerate.go @@ -23,7 +23,7 @@ import ( "log" "os" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/putobject-streaming.go b/examples/s3/putobject-streaming.go index bf7c5cc62..a90e4720f 100644 --- a/examples/s3/putobject-streaming.go +++ b/examples/s3/putobject-streaming.go @@ -23,7 +23,7 @@ import ( "log" "os" - minio "github.com/minio/minio-go" + minio "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/putobject.go b/examples/s3/putobject.go index dd4a024d0..31969e035 100644 --- a/examples/s3/putobject.go +++ b/examples/s3/putobject.go @@ -23,7 +23,7 @@ import ( "log" "os" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/removeallbucketnotification.go b/examples/s3/removeallbucketnotification.go index 9149bc133..afd8ce13b 100644 --- a/examples/s3/removeallbucketnotification.go +++ b/examples/s3/removeallbucketnotification.go @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/removebucket.go b/examples/s3/removebucket.go index 066eb1f7c..7f2295e6a 100644 --- a/examples/s3/removebucket.go +++ b/examples/s3/removebucket.go @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/removeincompleteupload.go b/examples/s3/removeincompleteupload.go index efc768fa3..9b941ea62 100644 --- a/examples/s3/removeincompleteupload.go +++ b/examples/s3/removeincompleteupload.go @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/removeobject.go b/examples/s3/removeobject.go index cdebd7190..967ec8c5f 100644 --- a/examples/s3/removeobject.go +++ b/examples/s3/removeobject.go @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/removeobjects.go b/examples/s3/removeobjects.go index 007c146b0..d453beea2 100644 --- a/examples/s3/removeobjects.go +++ b/examples/s3/removeobjects.go @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/selectobject.go b/examples/s3/selectobject.go index a6985fc74..e7a837036 100644 --- a/examples/s3/selectobject.go +++ b/examples/s3/selectobject.go @@ -25,7 +25,7 @@ import ( "log" "os" - minio "github.com/minio/minio-go" + minio "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/setbucketlifecycle.go b/examples/s3/setbucketlifecycle.go index b569e7403..362caa8c9 100644 --- a/examples/s3/setbucketlifecycle.go +++ b/examples/s3/setbucketlifecycle.go @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/setbucketnotification.go b/examples/s3/setbucketnotification.go index 5e644644e..bb6ab5e76 100644 --- a/examples/s3/setbucketnotification.go +++ b/examples/s3/setbucketnotification.go @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/setbucketpolicy.go b/examples/s3/setbucketpolicy.go index 8ab169799..607627865 100644 --- a/examples/s3/setbucketpolicy.go +++ b/examples/s3/setbucketpolicy.go @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/statobject.go b/examples/s3/statobject.go index b20a84199..9e8fec65d 100644 --- a/examples/s3/statobject.go +++ b/examples/s3/statobject.go @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/functional_tests.go b/functional_tests.go index b412edb23..865c2404c 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -40,10 +40,10 @@ import ( "time" humanize "github.com/dustin/go-humanize" - minio "github.com/minio/minio-go" log "github.com/sirupsen/logrus" - "github.com/minio/minio-go/pkg/encrypt" + "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v6/pkg/encrypt" ) const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" diff --git a/go.mod b/go.mod index 59ef0a794..9d342468d 100644 --- a/go.mod +++ b/go.mod @@ -1,14 +1,16 @@ -module github.com/minio/minio-go +module github.com/minio/minio-go/v6 + +go 1.12 require ( - github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e // indirect - github.com/jtolds/gls v4.2.1+incompatible // indirect + github.com/a8m/mark v0.1.1-0.20170507133748-44f2db618845 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/gernest/wow v0.1.0 // indirect + github.com/minio/cli v1.20.0 // indirect github.com/mitchellh/go-homedir v1.1.0 - github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 // indirect - github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c // indirect - golang.org/x/crypto v0.0.0-20190128193316-c7b33c32a30b - golang.org/x/net v0.0.0-20190213061140-3a22650c66bd - golang.org/x/sys v0.0.0-20190124100055-b90733256f2e // indirect - golang.org/x/text v0.3.0 // indirect - gopkg.in/ini.v1 v1.41.0 + github.com/sirupsen/logrus v1.4.2 // indirect + github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect + golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f + golang.org/x/net v0.0.0-20190522155817-f3200d17e092 + gopkg.in/ini.v1 v1.42.0 ) diff --git a/go.sum b/go.sum index baf2f44e8..62a4f31fa 100644 --- a/go.sum +++ b/go.sum @@ -1,20 +1,48 @@ -github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg= -github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= -github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/a8m/mark v0.1.1-0.20170507133748-44f2db618845 h1:hIjQrEARcc9LcH8igte3JBpWBZ7+SpinU70dOjU/afo= +github.com/a8m/mark v0.1.1-0.20170507133748-44f2db618845/go.mod h1:c8Mh99Cw82nrsAnPgxQSZHkswVOJF7/MqZb1ZdvriLM= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/gernest/wow v0.1.0 h1:g9xdwCwP0+xgVYlA2sopI0gZHqXe7HjI/7/LykG4fks= +github.com/gernest/wow v0.1.0/go.mod h1:dEPabJRi5BneI1Nev1VWo0ZlcTWibHWp43qxKms4elY= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/minio/cli v1.20.0 h1:OVNIt8Rg5+mpYb8siWT2gBV5hvUyFbRvBikC+Ytvf5A= +github.com/minio/cli v1.20.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 h1:Jpy1PXuP99tXNrhbq2BaPz9B+jNAvH1JPQQpG/9GCXY= -github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c h1:Ho+uVpkel/udgjbwB5Lktg9BtvJSh2DT0Hi6LPSyI2w= -github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= -golang.org/x/crypto v0.0.0-20190128193316-c7b33c32a30b h1:Ib/yptP38nXZFMwqWSip+OKuMP9OkyDe3p+DssP8n9w= -golang.org/x/crypto v0.0.0-20190128193316-c7b33c32a30b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd h1:HuTn7WObtcDo9uEEU7rEqL0jYthdXAmZ6PP+meazmaU= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/sys v0.0.0-20190124100055-b90733256f2e h1:3GIlrlVLfkoipSReOMNAgApI0ajnalyLa/EZHHca/XI= -golang.org/x/sys v0.0.0-20190124100055-b90733256f2e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo= +golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/sys v0.0.0-20190116161447-11f53e031339/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -gopkg.in/ini.v1 v1.41.0 h1:Ka3ViY6gNYSKiVy71zXBEqKplnV35ImDLVG+8uoIklE= -gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= +gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/credentials/iam_aws_test.go b/pkg/credentials/iam_aws_test.go index cc5518352..b33f9bd50 100644 --- a/pkg/credentials/iam_aws_test.go +++ b/pkg/credentials/iam_aws_test.go @@ -1,3 +1,5 @@ +// +build !windows + /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017 MinIO, Inc. @@ -69,7 +71,7 @@ func initTestServer(expireOn string, failAssume bool) *httptest.Server { fmt.Fprintln(w, "RoleName") } else if r.URL.Path == "/latest/meta-data/iam/security-credentials/RoleName" { if failAssume { - fmt.Fprintf(w, credsFailRespTmpl) + fmt.Fprint(w, credsFailRespTmpl) } else { fmt.Fprintf(w, credsRespTmpl, expireOn) } diff --git a/pkg/policy/bucket-policy-condition.go b/pkg/policy/bucket-policy-condition.go index c5eb67238..b256faff4 100644 --- a/pkg/policy/bucket-policy-condition.go +++ b/pkg/policy/bucket-policy-condition.go @@ -17,7 +17,7 @@ package policy -import "github.com/minio/minio-go/pkg/set" +import "github.com/minio/minio-go/v6/pkg/set" // ConditionKeyMap - map of policy condition key and value. type ConditionKeyMap map[string]set.StringSet diff --git a/pkg/policy/bucket-policy-condition_test.go b/pkg/policy/bucket-policy-condition_test.go index e1cc2f3d1..2fc9baa60 100644 --- a/pkg/policy/bucket-policy-condition_test.go +++ b/pkg/policy/bucket-policy-condition_test.go @@ -21,7 +21,7 @@ import ( "encoding/json" "testing" - "github.com/minio/minio-go/pkg/set" + "github.com/minio/minio-go/v6/pkg/set" ) // ConditionKeyMap.Add() is called and the result is validated. diff --git a/pkg/policy/bucket-policy.go b/pkg/policy/bucket-policy.go index c3dd83d6a..f2c728924 100644 --- a/pkg/policy/bucket-policy.go +++ b/pkg/policy/bucket-policy.go @@ -23,7 +23,7 @@ import ( "reflect" "strings" - "github.com/minio/minio-go/pkg/set" + "github.com/minio/minio-go/v6/pkg/set" ) // BucketPolicy - Bucket level policy. diff --git a/pkg/policy/bucket-policy_test.go b/pkg/policy/bucket-policy_test.go index bd3ae59ff..a514eaa0f 100644 --- a/pkg/policy/bucket-policy_test.go +++ b/pkg/policy/bucket-policy_test.go @@ -23,7 +23,7 @@ import ( "reflect" "testing" - "github.com/minio/minio-go/pkg/set" + "github.com/minio/minio-go/v6/pkg/set" ) // TestUnmarshalBucketPolicy tests unmarsheling various examples diff --git a/pkg/s3signer/request-signature-v2.go b/pkg/s3signer/request-signature-v2.go index 919540690..40ba07130 100644 --- a/pkg/s3signer/request-signature-v2.go +++ b/pkg/s3signer/request-signature-v2.go @@ -30,7 +30,7 @@ import ( "strings" "time" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // Signature and API related constants. diff --git a/pkg/s3signer/request-signature-v4.go b/pkg/s3signer/request-signature-v4.go index e23c06baf..ab96b58c5 100644 --- a/pkg/s3signer/request-signature-v4.go +++ b/pkg/s3signer/request-signature-v4.go @@ -26,7 +26,7 @@ import ( "strings" "time" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // Signature and API related constants. diff --git a/pkg/set/stringset_test.go b/pkg/set/stringset_test.go index aeda4e231..12aa6f90a 100644 --- a/pkg/set/stringset_test.go +++ b/pkg/set/stringset_test.go @@ -127,7 +127,7 @@ func TestStringSetFuncMatch(t *testing.T) { }{ // Test to check match function doing case insensive compare. {func(setValue string, compareValue string) bool { - return strings.ToUpper(setValue) == strings.ToUpper(compareValue) + return strings.EqualFold(setValue, compareValue) }, "Bar", `[bar]`}, // Test to check match function doing prefix check. {func(setValue string, compareValue string) bool { diff --git a/staticcheck.conf b/staticcheck.conf new file mode 100644 index 000000000..71cc6f536 --- /dev/null +++ b/staticcheck.conf @@ -0,0 +1 @@ +checks = ["all", "-ST1005", "-ST1017", "-SA9004", "-ST1000", "-S1021"] \ No newline at end of file diff --git a/utils.go b/utils.go index dbad3b103..fc30c1ab7 100644 --- a/utils.go +++ b/utils.go @@ -32,7 +32,7 @@ import ( "strings" "time" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // xmlDecoder provide decoded value in xml. @@ -229,7 +229,7 @@ var supportedHeaders = []string{ // isStorageClassHeader returns true if the header is a supported storage class header func isStorageClassHeader(headerKey string) bool { - return strings.ToLower(amzStorageClass) == strings.ToLower(headerKey) + return strings.EqualFold(amzStorageClass, headerKey) } // isStandardHeader returns true if header is a supported header and not a custom header diff --git a/utils_test.go b/utils_test.go index 57ce9d258..0ce163812 100644 --- a/utils_test.go +++ b/utils_test.go @@ -24,7 +24,7 @@ import ( "testing" "time" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // Tests signature redacting function used From c5ac8248d5e1f0a70329451f0a90209e9c1d5e62 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 28 May 2019 15:24:59 -0700 Subject: [PATCH 027/215] Increase default multipart size to 128MiB (#1112) --- .travis.yml | 17 ++++++++++--- Makefile | 4 +-- api-put-object-common.go | 2 +- api-put-object-streaming.go | 2 +- api-put-object.go | 6 ++--- api_unit_test.go | 20 +++++++-------- constants.go | 4 +-- docs/API.md | 4 +-- docs/zh_CN/API.md | 4 +-- functional_tests.go | 50 ++++++++++++++++++------------------- testcerts/private.key | 28 +++++++++++++++++++++ testcerts/public.crt | 25 +++++++++++++++++++ 12 files changed, 114 insertions(+), 52 deletions(-) create mode 100644 testcerts/private.key create mode 100644 testcerts/public.crt diff --git a/.travis.yml b/.travis.yml index 7adb883ff..bc891f3fb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,6 @@ os: env: - ARCH=x86_64 -- ARCH=i686 go: - 1.12.x @@ -22,7 +21,17 @@ addons: packages: - devscripts +before_install: + - curl -O https://dl.minio.io/server/minio/release/linux-amd64/minio && chmod +x ./minio + - sudo cp testcerts/public.crt /usr/local/share/ca-certificates/ + - sudo update-ca-certificates + - MINIO_ACCESS_KEY=minio MINIO_SECRET_KEY=minio123 ./minio server --compat --quiet --certs-dir testcerts data 2>&1 > minio.log & + script: -- diff -au <(gofmt -d .) <(printf "") -- diff -au <(licensecheck --check '.go$' --recursive --lines 0 * | grep -v -w 'Apache (v2.0)') <(printf "") -- make + - diff -au <(gofmt -d .) <(printf "") + - diff -au <(licensecheck --check '.go$' --recursive --lines 0 * | grep -v -w 'Apache (v2.0)') <(printf "") + - make + +addons: + artifacts: + - minio.log diff --git a/Makefile b/Makefile index fa0936c0f..892f8cf50 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ vet: @GO111MODULE=on go vet ./... test: - @GO111MODULE=on SERVER_ENDPOINT=play.min.io:9000 ACCESS_KEY=Q3AM3UQ867SPQQA43P2F SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG ENABLE_HTTPS=1 MINT_MODE=full go test -race -v ./... + @GO111MODULE=on SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minio SECRET_KEY=minio123 ENABLE_HTTPS=1 MINT_MODE=full go test -race -v ./... examples: @mkdir -p /tmp/examples && for i in $(echo examples/s3/*); do go build -o /tmp/examples/$(basename ${i:0:-3}) ${i}; done @@ -17,4 +17,4 @@ docs: @(cd docs; GO111MODULE=on go build validator.go && ./validator -m ../docs/API.md -t checker.go.tpl) functional-test: - @GO111MODULE=on SERVER_ENDPOINT=play.min.io:9000 ACCESS_KEY=Q3AM3UQ867SPQQA43P2F SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG ENABLE_HTTPS=1 MINT_MODE=full go run functional_tests.go + @GO111MODULE=on SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minio SECRET_KEY=minio123 ENABLE_HTTPS=1 MINT_MODE=full go run functional_tests.go diff --git a/api-put-object-common.go b/api-put-object-common.go index 0312fd317..a786d2a8e 100644 --- a/api-put-object-common.go +++ b/api-put-object-common.go @@ -64,7 +64,7 @@ func isReadAt(reader io.Reader) (ok bool) { // object storage it will have the following parameters as constants. // // maxPartsCount - 10000 -// minPartSize - 64MiB +// minPartSize - 128MiB // maxMultipartPutObjectSize - 5TiB // func optimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCount int, partSize int64, lastPartSize int64, err error) { diff --git a/api-put-object-streaming.go b/api-put-object-streaming.go index 8bea9b4b8..0d3f2455a 100644 --- a/api-put-object-streaming.go +++ b/api-put-object-streaming.go @@ -75,7 +75,7 @@ type uploadPartReq struct { Part *ObjectPart // Size of the part uploaded. } -// putObjectMultipartFromReadAt - Uploads files bigger than 64MiB. +// putObjectMultipartFromReadAt - Uploads files bigger than 128MiB. // Supports all readers which implements io.ReaderAt interface // (ReadAt method). // diff --git a/api-put-object.go b/api-put-object.go index 0ae7e4c69..8f33fca6c 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -124,9 +124,9 @@ func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].Part // // You must have WRITE permissions on a bucket to create an object. // -// - For size smaller than 64MiB PutObject automatically does a +// - For size smaller than 128MiB PutObject automatically does a // single atomic Put operation. -// - For size larger than 64MiB PutObject automatically does a +// - For size larger than 128MiB PutObject automatically does a // multipart Put operation. // - For size input as -1 PutObject does a multipart Put operation // until input stream reaches EOF. Maximum object size that can @@ -167,7 +167,7 @@ func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName stri return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts) } - // For all sizes greater than 64MiB do multipart. + // For all sizes greater than 128MiB do multipart. return c.putObjectMultipartStream(ctx, bucketName, objectName, reader, size, opts) } diff --git a/api_unit_test.go b/api_unit_test.go index c01578052..fbfccdbb8 100644 --- a/api_unit_test.go +++ b/api_unit_test.go @@ -120,11 +120,11 @@ func TestPartSize(t *testing.T) { if err != nil { t.Fatal("Error: ", err) } - if totalPartsCount != 79 { - t.Fatalf("Error: expecting total parts count of 79: got %v instead", totalPartsCount) + if totalPartsCount != 40 { + t.Fatalf("Error: expecting total parts count of 40: got %v instead", totalPartsCount) } - if partSize != 67108864 { - t.Fatalf("Error: expecting part size of 67108864: got %v instead", partSize) + if partSize != 134217728 { + t.Fatalf("Error: expecting part size of 134217728: got %v instead", partSize) } if lastPartSize != 9437184 { t.Fatalf("Error: expecting last part size of 9437184: got %v instead", lastPartSize) @@ -146,14 +146,14 @@ func TestPartSize(t *testing.T) { if err != nil { t.Fatal("Error:", err) } - if totalPartsCount != 9103 { - t.Fatalf("Error: expecting total parts count of 9103: got %v instead", totalPartsCount) + if totalPartsCount != 8192 { + t.Fatalf("Error: expecting total parts count of 8192: got %v instead", totalPartsCount) } - if partSize != 603979776 { - t.Fatalf("Error: expecting part size of 603979776: got %v instead", partSize) + if partSize != 671088640 { + t.Fatalf("Error: expecting part size of 671088640: got %v instead", partSize) } - if lastPartSize != 134217728 { - t.Fatalf("Error: expecting last part size of 134217728: got %v instead", lastPartSize) + if lastPartSize != 671088640 { + t.Fatalf("Error: expecting last part size of 671088640: got %v instead", lastPartSize) } } diff --git a/constants.go b/constants.go index 252fc5faf..ac472a631 100644 --- a/constants.go +++ b/constants.go @@ -23,9 +23,9 @@ package minio // a part in a multipart upload may not be uploaded. const absMinPartSize = 1024 * 1024 * 5 -// minPartSize - minimum part size 64MiB per object after which +// minPartSize - minimum part size 128MiB per object after which // putObject behaves internally as multipart. -const minPartSize = 1024 * 1024 * 64 +const minPartSize = 1024 * 1024 * 128 // maxPartsCount - maximum number of parts for a single multipart session. const maxPartsCount = 10000 diff --git a/docs/API.md b/docs/API.md index 7f580cfa5..6e24ed91c 100644 --- a/docs/API.md +++ b/docs/API.md @@ -528,7 +528,7 @@ if err != nil { ### PutObject(bucketName, objectName string, reader io.Reader, objectSize int64,opts PutObjectOptions) (n int, err error) -Uploads objects that are less than 64MiB in a single PUT operation. For objects that are greater than 64MiB in size, PutObject seamlessly uploads the object as parts of 64MiB or more depending on the actual file size. The max upload size for an object is 5TB. +Uploads objects that are less than 128MiB in a single PUT operation. For objects that are greater than 128MiB in size, PutObject seamlessly uploads the object as parts of 128MiB or more depending on the actual file size. The max upload size for an object is 5TB. __Parameters__ @@ -869,7 +869,7 @@ if err != nil { ### FPutObject(bucketName, objectName, filePath, opts PutObjectOptions) (length int64, err error) Uploads contents from a file to objectName. -FPutObject uploads objects that are less than 64MiB in a single PUT operation. For objects that are greater than the 64MiB in size, FPutObject seamlessly uploads the object in chunks of 64MiB or more depending on the actual file size. The max upload size for an object is 5TB. +FPutObject uploads objects that are less than 128MiB in a single PUT operation. For objects that are greater than the 128MiB in size, FPutObject seamlessly uploads the object in chunks of 128MiB or more depending on the actual file size. The max upload size for an object is 5TB. __Parameters__ diff --git a/docs/zh_CN/API.md b/docs/zh_CN/API.md index f6d84032f..a3778447a 100644 --- a/docs/zh_CN/API.md +++ b/docs/zh_CN/API.md @@ -550,7 +550,7 @@ if err != nil { ### PutObject(bucketName, objectName string, reader io.Reader, objectSize int64,opts PutObjectOptions) (n int, err error) -当对象小于64MiB时,直接在一次PUT请求里进行上传。当大于64MiB时,根据文件的实际大小,PutObject会自动地将对象进行拆分成64MiB一块或更大一些进行上传。对象的最大大小是5TB。 +当对象小于128MiB时,直接在一次PUT请求里进行上传。当大于128MiB时,根据文件的实际大小,PutObject会自动地将对象进行拆分成128MiB一块或更大一些进行上传。对象的最大大小是5TB。 __参数__ @@ -889,7 +889,7 @@ if err != nil { ### FPutObject(bucketName, objectName, filePath, opts PutObjectOptions) (length int64, err error) 将filePath对应的文件内容上传到一个对象中。 -当对象小于64MiB时,FPutObject直接在一次PUT请求里进行上传。当大于64MiB时,根据文件的实际大小,FPutObject会自动地将对象进行拆分成64MiB一块或更大一些进行上传。对象的最大大小是5TB。 +当对象小于128MiB时,FPutObject直接在一次PUT请求里进行上传。当大于128MiB时,根据文件的实际大小,FPutObject会自动地将对象进行拆分成128MiB一块或更大一些进行上传。对象的最大大小是5TB。 __参数__ diff --git a/functional_tests.go b/functional_tests.go index 865c2404c..d3d116f4e 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -264,7 +264,7 @@ var dataFileMap = map[string]int{ "datafile-5-MB": 5 * humanize.MiByte, "datafile-6-MB": 6 * humanize.MiByte, "datafile-11-MB": 11 * humanize.MiByte, - "datafile-65-MB": 65 * humanize.MiByte, + "datafile-129-MB": 129 * humanize.MiByte, } func isFullMode() bool { @@ -531,8 +531,8 @@ func testPutObjectReadAt() { return } - bufSize := dataFileMap["datafile-65-MB"] - var reader = getDataReader("datafile-65-MB") + bufSize := dataFileMap["datafile-129-MB"] + var reader = getDataReader("datafile-129-MB") defer reader.Close() // Save the data @@ -641,8 +641,8 @@ func testPutObjectWithMetadata() { return } - bufSize := dataFileMap["datafile-65-MB"] - var reader = getDataReader("datafile-65-MB") + bufSize := dataFileMap["datafile-129-MB"] + var reader = getDataReader("datafile-129-MB") defer reader.Close() // Save the data @@ -1308,7 +1308,7 @@ func testFPutObjectMultipart() { } // Upload 4 parts to utilize all 3 'workers' in multipart and still have a part to upload. - var fileName = getMintDataDirFilePath("datafile-65-MB") + var fileName = getMintDataDirFilePath("datafile-129-MB") if fileName == "" { // Make a temp file with minPartSize bytes of data. file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") @@ -1317,7 +1317,7 @@ func testFPutObjectMultipart() { return } // Upload 2 parts to utilize all 3 'workers' in multipart and still have a part to upload. - if _, err = io.Copy(file, getDataReader("datafile-65-MB")); err != nil { + if _, err = io.Copy(file, getDataReader("datafile-129-MB")); err != nil { logError(testName, function, args, startTime, "", "Copy failed", err) return } @@ -1328,7 +1328,7 @@ func testFPutObjectMultipart() { fileName = file.Name() args["fileName"] = fileName } - totalSize := dataFileMap["datafile-65-MB"] + totalSize := dataFileMap["datafile-129-MB"] // Set base object name objectName := bucketName + "FPutObject" + "-standard" args["objectName"] = objectName @@ -1426,7 +1426,7 @@ func testFPutObject() { // Upload 3 parts worth of data to use all 3 of multiparts 'workers' and have an extra part. // Use different data in part for multipart tests to check parts are uploaded in correct order. - var fName = getMintDataDirFilePath("datafile-65-MB") + var fName = getMintDataDirFilePath("datafile-129-MB") if fName == "" { // Make a temp file with minPartSize bytes of data. file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") @@ -1436,7 +1436,7 @@ func testFPutObject() { } // Upload 3 parts to utilize all 3 'workers' in multipart and still have a part to upload. - if _, err = io.Copy(file, getDataReader("datafile-65-MB")); err != nil { + if _, err = io.Copy(file, getDataReader("datafile-129-MB")); err != nil { logError(testName, function, args, startTime, "", "File copy failed", err) return } @@ -1448,7 +1448,7 @@ func testFPutObject() { defer os.Remove(file.Name()) fName = file.Name() } - totalSize := dataFileMap["datafile-65-MB"] + totalSize := dataFileMap["datafile-129-MB"] // Set base object name function = "FPutObject(bucketName, objectName, fileName, opts)" @@ -2706,9 +2706,9 @@ func testSSECEncryptedGetObjectReadSeekFunctional() { } }() - // Generate 65MiB of data. - bufSize := dataFileMap["datafile-65-MB"] - var reader = getDataReader("datafile-65-MB") + // Generate 129MiB of data. + bufSize := dataFileMap["datafile-129-MB"] + var reader = getDataReader("datafile-129-MB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") @@ -2894,9 +2894,9 @@ func testSSES3EncryptedGetObjectReadSeekFunctional() { } }() - // Generate 65MiB of data. - bufSize := dataFileMap["datafile-65-MB"] - var reader = getDataReader("datafile-65-MB") + // Generate 129MiB of data. + bufSize := dataFileMap["datafile-129-MB"] + var reader = getDataReader("datafile-129-MB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") @@ -3072,9 +3072,9 @@ func testSSECEncryptedGetObjectReadAtFunctional() { return } - // Generate 65MiB of data. - bufSize := dataFileMap["datafile-65-MB"] - var reader = getDataReader("datafile-65-MB") + // Generate 129MiB of data. + bufSize := dataFileMap["datafile-129-MB"] + var reader = getDataReader("datafile-129-MB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") @@ -3263,9 +3263,9 @@ func testSSES3EncryptedGetObjectReadAtFunctional() { return } - // Generate 65MiB of data. - bufSize := dataFileMap["datafile-65-MB"] - var reader = getDataReader("datafile-65-MB") + // Generate 129MiB of data. + bufSize := dataFileMap["datafile-129-MB"] + var reader = getDataReader("datafile-129-MB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") @@ -8561,8 +8561,8 @@ func testPutObjectNoLengthV2() { objectName := bucketName + "unique" args["objectName"] = objectName - bufSize := dataFileMap["datafile-65-MB"] - var reader = getDataReader("datafile-65-MB") + bufSize := dataFileMap["datafile-129-MB"] + var reader = getDataReader("datafile-129-MB") defer reader.Close() args["size"] = bufSize diff --git a/testcerts/private.key b/testcerts/private.key new file mode 100644 index 000000000..09448cfb2 --- /dev/null +++ b/testcerts/private.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCwUyKC2VOXy2+8 +gMQkRrDJ4aA7K5pgj6LHWu25GeY93x+8DLFyQ9BhoaMcAbs2Cmw91rONDrZ0gNql +yi5JX8t+iiVH8o6dcq6W8jNLnOw0GMNJ2/E1Ckfe5ktkn9synSSwMdnFp3cDk7Hb +2j6IiWrb+PXb7VGL47kDrG59iKQ350MiB3PNpd1ulHbi2m2ZC3WyoTTzlgeTXiXa +zhBIX4wsGVYs6RzS1bTZFBq05dIPNMJCRDVBSBYAAVuBxKjh4xvhC6j0rTCCK8uJ +752KioW4Y0VAEv6yUC4Ht6D9Jcj7gODTgb2irWSCNXFH+pZaI6wWlS8pPiL6iljY +P3kBeFiLAgMBAAECggEAKM20SM9+FryPSPILcdGiC7XY3JiEix/yLWwPYyxpKZw+ +vce6MJUc3dsH4e1Mo37Z+Z17w4LKGj/PWVpmR7iRYOEbK4EoG6t0V54I3NCdoJiy +aJ8rPHj6lMx6WfjcQuQ2n0eJ+8F7OyqsmBHzMqmKPwln69MJcfPq1rzKfOZoCj9p +0oZ+3Iv3roC4uH8peZFooCDUlzJL+8KiybVlemNfklKsHfRmL2vOdFBt+qvit6N/ +9JgBTX1mRx1+vqECj+TlVP//k3BTEPNfpIvsLCRN0eBbQcXYzu/gZfHwGnsy5Lxy +HaHNJnmLZMWSCc4iyCK7uN/BHXNUSSh3qqp4wqz0IQKBgQDdGbOuOVdJW4J1yYua +nDLAu2RQqvZTnoz1/b8jcrjS7tS1la5x1IN0Z9/VqTmkfxyHOK9ab1iVlklkIzjP +CmHnadUwr8vrLcdicFpjVLQU3O4ZqGrgiSGIPAotvOfAOuuzMs+r5ElW/MrGq0Pa +/3tGCTIx8JscZZjGhffUNoIGeQKBgQDMKB+flQB9Ajeo1JM4y3DtHbMJ5D2+/qoe +IkM7kN5K85EEpNwA2PMNKL2qthgM9YFU3K6Dj0gxPNsUKg3W7Ff2r+gaj8K+VjU0 +VbdhTZANbou8hU551swDUCUgquassMtZJIdZnQ7puwLGK67sZwWlOS6Pe1aqaNc5 +nY/MRbemIwKBgEySfykCkNlGCPuUDnZATE91VrudSewRyA3VkGHNdHcQ4bf1m9Gu +YMxqwRl1HxJ6Nz4ZgplWYJ6FyusUS7NgjCGiBIR1DbFoTFoqQROPnUJwdUGLk2Ap +/eP5ryjB+J0ZitGn8kY8rK2kpPGDFN/+hQnvW2PySTXfdbajZP4o1oU5AoGAMiT0 +x3yQlyPRSf2Uf5Gwlf0Ceb5+0Ae6/xXJT7sgbmZuyyY3B1pCMIw+MczyEVTHxHFD +x/qMb9OTt9swdQauAGBqcQO4gImqHcWj+hlT9Yied9qCUPjKOVIZHHH9oJL4D1gi +iodCH3SYlNYr69LOFyv5XLKdsdN4caVaqYDCP+MCgYEAwXyCmSml5oCxJeAOrEDC +Yg3vq3Ul9JO1wc8VDXn9+DtnFsuRHm0fTIxBelmis8AjIIq5DcObpk6wGYZwUiTU +LYQU7v0/Azujv9cl10GI8wzYKiRvExZDTn0sp6OKnau735qBUZvsRDqEQQ5n7waZ +xjlGmZyfah17laYZV9aJoHk= +-----END PRIVATE KEY----- diff --git a/testcerts/public.crt b/testcerts/public.crt new file mode 100644 index 000000000..71f4ccc65 --- /dev/null +++ b/testcerts/public.crt @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIEKjCCApKgAwIBAgIRAPVKnAiFmDti207oQPs2VfUwDQYJKoZIhvcNAQELBQAw +VTEeMBwGA1UEChMVbWtjZXJ0IGRldmVsb3BtZW50IENBMRUwEwYDVQQLDAxoYXJz +aGFAcHJ0c2MxHDAaBgNVBAMME21rY2VydCBoYXJzaGFAcHJ0c2MwHhcNMTkwMTA3 +MTE1ODE2WhcNMjkwMTA3MTE1ODE2WjBEMScwJQYDVQQKEx5ta2NlcnQgZGV2ZWxv +cG1lbnQgY2VydGlmaWNhdGUxGTAXBgNVBAsMEGhhcnNoYUBiYWNrc3BhY2UwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCwUyKC2VOXy2+8gMQkRrDJ4aA7 +K5pgj6LHWu25GeY93x+8DLFyQ9BhoaMcAbs2Cmw91rONDrZ0gNqlyi5JX8t+iiVH +8o6dcq6W8jNLnOw0GMNJ2/E1Ckfe5ktkn9synSSwMdnFp3cDk7Hb2j6IiWrb+PXb +7VGL47kDrG59iKQ350MiB3PNpd1ulHbi2m2ZC3WyoTTzlgeTXiXazhBIX4wsGVYs +6RzS1bTZFBq05dIPNMJCRDVBSBYAAVuBxKjh4xvhC6j0rTCCK8uJ752KioW4Y0VA +Ev6yUC4Ht6D9Jcj7gODTgb2irWSCNXFH+pZaI6wWlS8pPiL6iljYP3kBeFiLAgMB +AAGjgYUwgYIwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwG +A1UdEwEB/wQCMAAwHwYDVR0jBBgwFoAUD575sRLoRt9dCxSRqbVctoEHt3MwLAYD +VR0RBCUwI4IJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAAAAABMA0GCSqG +SIb3DQEBCwUAA4IBgQC7qDRDNAHtfGtQs1UmvqWvHPI7qcBQgAibYq/Fox6X9ia1 +weQBfNWEoNOsk97wzbTz81ifXIQ0oV11kWE8EdsbXOf9xeFe9FmDn10d4bGjuMLd ++N3OtGKxLWry2xDYEsVHJZxVxwrf5GK6AJSJj/S837Nil6uRuwjvBVTbxmh1q0nV +x63V8Ag65rLS0fu8msSb64N5UHMCQk6IE+BFHY2gh0lBfZHMdtP4IbeCm756K78/ +WMeqjavGA3bqzVTixCHnJ9S2VLk/oQUS6mL869jM8+tN5VeE6Qsr1/Q5h+NaFCJg +Ed5xjT9mmnc3BLsOHflb1dg+rA90Zz9wphgebXbJhRNuuDRv81dtRPTzM+evGRGM +iRKtiDpog+K0HulfX2g4ZQ1dItEjYz+JYgUFJG+yCvBlNZ/WsTrIVcUCFKaG5rUC +aNqvKrSXfbzKQx7V/TtUAeSfRk7TBRn5qh8Pl+MmQQsB0L9hwTdnqTNn057tghu4 +3/yIIBpzdWPhQ5uv7Vc= +-----END CERTIFICATE----- From 6302adda1280560f8f13f57abba9e86880af2b63 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Wed, 29 May 2019 01:57:51 +0000 Subject: [PATCH 028/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 0c358f368..ece498f41 100644 --- a/api.go +++ b/api.go @@ -103,7 +103,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.26" + libraryVersion = "v6.0.27" ) // User Agent should always following the below style. From de69c0e465ed3608f3be32f2488af7b481b88e53 Mon Sep 17 00:00:00 2001 From: Matt Cottingham Date: Wed, 29 May 2019 16:25:32 +0100 Subject: [PATCH 029/215] Retry errors caused by a timeout waiting for the server response (#1116) --- retry.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/retry.go b/retry.go index 94176b0a1..2c608bab5 100644 --- a/retry.go +++ b/retry.go @@ -111,6 +111,9 @@ func isHTTPReqErrorRetryable(err error) bool { } else if strings.Contains(err.Error(), "net/http: HTTP/1.x transport connection broken") { // If error is transport connection broken, retry. return true + } else if strings.Contains(err.Error(), "net/http: timeout awaiting response headers") { + // Retry errors due to server not sending the response before timeout + return true } } return false From 952dac37c16b1c249672e0127f56c4927a2d9d29 Mon Sep 17 00:00:00 2001 From: Shern Shiou Tan Date: Wed, 29 May 2019 19:10:01 +0200 Subject: [PATCH 030/215] Add slash to defaultIAMSecurityCredsPath (#1115) Also add slash to security credential url at test --- pkg/credentials/iam_aws.go | 2 +- pkg/credentials/iam_aws_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/credentials/iam_aws.go b/pkg/credentials/iam_aws.go index 310785209..5732f2e4b 100644 --- a/pkg/credentials/iam_aws.go +++ b/pkg/credentials/iam_aws.go @@ -53,7 +53,7 @@ type IAM struct { const ( defaultIAMRoleEndpoint = "http://169.254.169.254" defaultECSRoleEndpoint = "http://169.254.170.2" - defaultIAMSecurityCredsPath = "/latest/meta-data/iam/security-credentials" + defaultIAMSecurityCredsPath = "/latest/meta-data/iam/security-credentials/" ) // https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html diff --git a/pkg/credentials/iam_aws_test.go b/pkg/credentials/iam_aws_test.go index b33f9bd50..90f980693 100644 --- a/pkg/credentials/iam_aws_test.go +++ b/pkg/credentials/iam_aws_test.go @@ -67,7 +67,7 @@ func initTestServerNoRoles() *httptest.Server { func initTestServer(expireOn string, failAssume bool) *httptest.Server { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/latest/meta-data/iam/security-credentials" { + if r.URL.Path == "/latest/meta-data/iam/security-credentials/" { fmt.Fprintln(w, "RoleName") } else if r.URL.Path == "/latest/meta-data/iam/security-credentials/RoleName" { if failAssume { From 1726b1f7de658861dfda481dd6b5e3da965ad089 Mon Sep 17 00:00:00 2001 From: kannappanr <30541348+kannappanr@users.noreply.github.com> Date: Wed, 29 May 2019 10:21:08 -0700 Subject: [PATCH 031/215] Listbuckets API: Retry with region returned by s3 server (#1113) In case of AuthorizationHeaderMalformed error, retry with the region returned by the S3 server. This allows listBuckets to work with S3 servers that does not support signing with us-east-1 by default --- api-list.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/api-list.go b/api-list.go index b9c0f5d8d..2fb0b6233 100644 --- a/api-list.go +++ b/api-list.go @@ -41,6 +41,23 @@ import ( func (c Client) ListBuckets() ([]BucketInfo, error) { // Execute GET on service. resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{contentSHA256Hex: emptySHA256Hex}) + + if err != nil { + closeResponse(resp) + return nil, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + errResponse := ToErrorResponse(httpRespToErrorResponse(resp, "", "")) + closeResponse(resp) + if errResponse.Code != "AuthorizationHeaderMalformed" && errResponse.Code != "InvalidRegion" { + return nil, httpRespToErrorResponse(resp, "", "") + } + // Retry with the region returned by the server. + resp, err = c.executeMethod(context.Background(), "GET", requestMetadata{contentSHA256Hex: emptySHA256Hex, bucketLocation: errResponse.Region}) + } + } + defer closeResponse(resp) if err != nil { return nil, err From e1cd915be1313f4bcb97a4fd841bc70f18b023c9 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Wed, 29 May 2019 21:13:12 +0000 Subject: [PATCH 032/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index ece498f41..67b76e4dc 100644 --- a/api.go +++ b/api.go @@ -103,7 +103,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.27" + libraryVersion = "v6.0.28" ) // User Agent should always following the below style. From 0c7b691852b3fe099961bd95a74fb957998e8eb7 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 5 Jun 2019 15:47:49 -0700 Subject: [PATCH 033/215] Handle regions returned in error responses (#1117) --- api-list.go | 17 ----------------- api.go | 20 ++++++++++++-------- bucket-cache.go | 12 ++++++++++-- 3 files changed, 22 insertions(+), 27 deletions(-) diff --git a/api-list.go b/api-list.go index 2fb0b6233..b9c0f5d8d 100644 --- a/api-list.go +++ b/api-list.go @@ -41,23 +41,6 @@ import ( func (c Client) ListBuckets() ([]BucketInfo, error) { // Execute GET on service. resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{contentSHA256Hex: emptySHA256Hex}) - - if err != nil { - closeResponse(resp) - return nil, err - } - if resp != nil { - if resp.StatusCode != http.StatusOK { - errResponse := ToErrorResponse(httpRespToErrorResponse(resp, "", "")) - closeResponse(resp) - if errResponse.Code != "AuthorizationHeaderMalformed" && errResponse.Code != "InvalidRegion" { - return nil, httpRespToErrorResponse(resp, "", "") - } - // Retry with the region returned by the server. - resp, err = c.executeMethod(context.Background(), "GET", requestMetadata{contentSHA256Hex: emptySHA256Hex, bucketLocation: errResponse.Region}) - } - } - defer closeResponse(resp) if err != nil { return nil, err diff --git a/api.go b/api.go index 67b76e4dc..0e83d8607 100644 --- a/api.go +++ b/api.go @@ -653,14 +653,23 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque // // Additionally we should only retry if bucketLocation and custom // region is empty. - if metadata.bucketLocation == "" && c.region == "" { - if errResponse.Code == "AuthorizationHeaderMalformed" || errResponse.Code == "InvalidRegion" { + if c.region == "" { + switch errResponse.Code { + case "AuthorizationHeaderMalformed": + fallthrough + case "InvalidRegion": + fallthrough + case "AccessDenied": if metadata.bucketName != "" && errResponse.Region != "" { // Gather Cached location only if bucketName is present. if _, cachedOk := c.bucketLocCache.Get(metadata.bucketName); cachedOk { c.bucketLocCache.Set(metadata.bucketName, errResponse.Region) continue // Retry. } + } else { + // Most probably for ListBuckets() + metadata.bucketLocation = errResponse.Region + continue // Retry } } } @@ -694,13 +703,8 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R // Gather location only if bucketName is present. location, err = c.getBucketLocation(metadata.bucketName) if err != nil { - if ToErrorResponse(err).Code != "AccessDenied" { - return nil, err - } + return nil, err } - // Upon AccessDenied error on fetching bucket location, default - // to possible locations based on endpoint URL. This can usually - // happen when GetBucketLocation() is disabled using IAM policies. } if location == "" { location = getDefaultLocation(*c.endpointURL, c.region) diff --git a/bucket-cache.go b/bucket-cache.go index 53a8a9c7f..4ea7e9181 100644 --- a/bucket-cache.go +++ b/bucket-cache.go @@ -124,8 +124,16 @@ func processBucketLocationResponse(resp *http.Response, bucketName string) (buck // For access denied error, it could be an anonymous // request. Move forward and let the top level callers // succeed if possible based on their policy. - if errResp.Code == "AccessDenied" { - return "us-east-1", nil + switch errResp.Code { + case "AuthorizationHeaderMalformed": + fallthrough + case "InvalidRegion": + fallthrough + case "AccessDenied": + if errResp.Region == "" { + return "us-east-1", nil + } + return errResp.Region, nil } return "", err } From 0cbba03df3cd4013a9afede363c73b35b1df7554 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Thu, 6 Jun 2019 00:49:37 +0000 Subject: [PATCH 034/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 0e83d8607..9acc59d6d 100644 --- a/api.go +++ b/api.go @@ -103,7 +103,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.28" + libraryVersion = "v6.0.29" ) // User Agent should always following the below style. From 91be771dfb0d41a709120c95a258bb22c66f87af Mon Sep 17 00:00:00 2001 From: Aditya Manthramurthy Date: Tue, 11 Jun 2019 13:40:48 -0700 Subject: [PATCH 035/215] Retry on AccessDenied only in restricted cases (#1121) List buckets should be retried on an AccessDenied error only if the region in the reponse is only different from the region in the request just made. --- api.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/api.go b/api.go index 9acc59d6d..f0719f2d7 100644 --- a/api.go +++ b/api.go @@ -668,8 +668,15 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque } } else { // Most probably for ListBuckets() - metadata.bucketLocation = errResponse.Region - continue // Retry + if errResponse.Region != metadata.bucketLocation { + // Retry if the error + // response has a + // different region + // than the request we + // just made. + metadata.bucketLocation = errResponse.Region + continue // Retry + } } } } From 916c055b844e8e47159555935642180f5aed9022 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Wed, 12 Jun 2019 19:28:30 +0000 Subject: [PATCH 036/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index f0719f2d7..ef90de9ab 100644 --- a/api.go +++ b/api.go @@ -103,7 +103,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.29" + libraryVersion = "v6.0.30" ) // User Agent should always following the below style. From f389a7273f8a9f2c803b85310a899da5fc66ba19 Mon Sep 17 00:00:00 2001 From: Daniel Valdivia Date: Fri, 21 Jun 2019 14:12:22 -0700 Subject: [PATCH 037/215] Prevent repeated allocation of constant value (#1128) --- api-notification.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/api-notification.go b/api-notification.go index a8d9fc613..f35619541 100644 --- a/api-notification.go +++ b/api-notification.go @@ -163,13 +163,14 @@ func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, even // Indicate to our routine to exit cleanly upon return. defer close(retryDoneCh) + // Prepare urlValues to pass into the request on every loop + urlValues := make(url.Values) + urlValues.Set("prefix", prefix) + urlValues.Set("suffix", suffix) + urlValues["events"] = events + // Wait on the jitter retry loop. for range c.newRetryTimerContinous(time.Second, time.Second*30, MaxJitter, retryDoneCh) { - urlValues := make(url.Values) - urlValues.Set("prefix", prefix) - urlValues.Set("suffix", suffix) - urlValues["events"] = events - // Execute GET on bucket to list objects. resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ bucketName: bucketName, From bb942b2c2341455b194a8b4ceeb3e6fd69255cf9 Mon Sep 17 00:00:00 2001 From: kannappanr <30541348+kannappanr@users.noreply.github.com> Date: Fri, 21 Jun 2019 16:39:01 -0700 Subject: [PATCH 038/215] Data race: Use a copy instead of a pointer to url.URL (#1126) --- bucket-cache.go | 2 +- bucket-cache_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bucket-cache.go b/bucket-cache.go index 4ea7e9181..7ba6cbb57 100644 --- a/bucket-cache.go +++ b/bucket-cache.go @@ -170,7 +170,7 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro urlValues.Set("location", "") // Set get bucket location always as path style. - targetURL := c.endpointURL + targetURL := *c.endpointURL // as it works in makeTargetURL method from api.go file if h, p, err := net.SplitHostPort(targetURL.Host); err == nil { diff --git a/bucket-cache_test.go b/bucket-cache_test.go index 8e4fe8fff..e095a5c0e 100644 --- a/bucket-cache_test.go +++ b/bucket-cache_test.go @@ -73,7 +73,7 @@ func TestGetBucketLocationRequest(t *testing.T) { urlValues.Set("location", "") // Set get bucket location always as path style. - targetURL := c.endpointURL + targetURL := *c.endpointURL targetURL.Path = path.Join(bucketName, "") + "/" targetURL.RawQuery = urlValues.Encode() From 481069ee84fa050b7b1c7c9456ec039c0f76b39a Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Wed, 26 Jun 2019 18:00:13 +0000 Subject: [PATCH 039/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index ef90de9ab..12be7a8e1 100644 --- a/api.go +++ b/api.go @@ -103,7 +103,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.30" + libraryVersion = "v6.0.31" ) // User Agent should always following the below style. From f5f036d4db301762d40c6b402fc2de3b3fbde0ed Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 26 Jun 2019 11:13:36 -0700 Subject: [PATCH 040/215] Fix travis failure --- .travis.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index bc891f3fb..5dbda5e7d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,12 +16,8 @@ matrix: allow_failures: - go: tip -addons: - apt: - packages: - - devscripts - before_install: + - sudo apt-get install devscripts - curl -O https://dl.minio.io/server/minio/release/linux-amd64/minio && chmod +x ./minio - sudo cp testcerts/public.crt /usr/local/share/ca-certificates/ - sudo update-ca-certificates @@ -31,7 +27,3 @@ script: - diff -au <(gofmt -d .) <(printf "") - diff -au <(licensecheck --check '.go$' --recursive --lines 0 * | grep -v -w 'Apache (v2.0)') <(printf "") - make - -addons: - artifacts: - - minio.log From 6d6e6371b0b60206739d3409341ee011eb45df01 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 26 Jun 2019 17:20:11 -0700 Subject: [PATCH 041/215] MakeBucket should always use pathStyle (#1129) --- api.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/api.go b/api.go index 12be7a8e1..64e72e298 100644 --- a/api.go +++ b/api.go @@ -719,10 +719,14 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R } // Look if target url supports virtual host. - isVirtualHost := c.isVirtualHostStyleRequest(*c.endpointURL, metadata.bucketName) + // We explicitly disallow MakeBucket calls to not use virtual DNS style, + // since the resolution may fail. + isMakeBucket := (metadata.objectName == "" && method == "PUT" && len(metadata.queryValues) == 0) + isVirtualHost := c.isVirtualHostStyleRequest(*c.endpointURL, metadata.bucketName) && !isMakeBucket // Construct a new target URL. - targetURL, err := c.makeTargetURL(metadata.bucketName, metadata.objectName, location, isVirtualHost, metadata.queryValues) + targetURL, err := c.makeTargetURL(metadata.bucketName, metadata.objectName, location, + isVirtualHost, metadata.queryValues) if err != nil { return nil, err } From 6898080dc99f7dbd177749e1baa4d2b8f0569ca6 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Thu, 27 Jun 2019 01:15:47 +0000 Subject: [PATCH 042/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 64e72e298..4be14ca51 100644 --- a/api.go +++ b/api.go @@ -103,7 +103,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.31" + libraryVersion = "v6.0.32" ) // User Agent should always following the below style. From 26dec1a6ef2e943908b058233dbd091b411696d5 Mon Sep 17 00:00:00 2001 From: SerJ Date: Wed, 10 Jul 2019 01:26:56 +0300 Subject: [PATCH 043/215] ReadAt should behave appropriately at io.EOF (#1138) Fixes #1137 --- api-get-object.go | 14 ++--- functional_tests.go | 128 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 6 deletions(-) diff --git a/api-get-object.go b/api-get-object.go index e4cd851f4..5d03ea9bb 100644 --- a/api-get-object.go +++ b/api-get-object.go @@ -321,6 +321,7 @@ func (o *Object) Read(b []byte) (n int, err error) { if o.prevErr != nil || o.isClosed { return 0, o.prevErr } + // Create a new request. readReq := getRequest{ isReadOp: true, @@ -403,10 +404,13 @@ func (o *Object) ReadAt(b []byte, offset int64) (n int, err error) { defer o.mutex.Unlock() // prevErr is error which was saved in previous operation. - if o.prevErr != nil || o.isClosed { + if o.prevErr != nil && o.prevErr != io.EOF || o.isClosed { return 0, o.prevErr } + // Set the current offset to ReadAt offset, because the current offset will be shifted at the end of this method. + o.currOffset = offset + // Can only compare offsets to size when size has been set. if o.objectInfoSet { // If offset is negative than we return io.EOF. @@ -476,11 +480,9 @@ func (o *Object) Seek(offset int64, whence int) (n int64, err error) { o.mutex.Lock() defer o.mutex.Unlock() - if o.prevErr != nil { - // At EOF seeking is legal allow only io.EOF, for any other errors we return. - if o.prevErr != io.EOF { - return 0, o.prevErr - } + // At EOF seeking is legal allow only io.EOF, for any other errors we return. + if o.prevErr != nil && o.prevErr != io.EOF { + return 0, o.prevErr } // Negative offset is valid for whence of '2'. diff --git a/functional_tests.go b/functional_tests.go index d3d116f4e..fc8143f10 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -2226,6 +2226,133 @@ func testGetObjectReadAtFunctional() { successLogger(testName, function, args, startTime).Info() } +// Reproduces issue https://github.com/minio/minio-go/issues/1137 +func testGetObjectReadAtWhenEOFWasReached() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "GetObject(bucketName, objectName)" + args := map[string]interface{}{} + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Generate 33K of data. + bufSize := dataFileMap["datafile-33-kB"] + var reader = getDataReader("datafile-33-kB") + defer reader.Close() + + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + buf, err := ioutil.ReadAll(reader) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAll failed", err) + return + } + + // Save the data + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + if n != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) + return + } + + // read the data back + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + // read directly + buf1 := make([]byte, n) + buf2 := make([]byte, 512) + + m, err := r.Read(buf1) + if err != nil { + if err != io.EOF { + logError(testName, function, args, startTime, "", "Read failed", err) + return + } + } + if m != len(buf1) { + logError(testName, function, args, startTime, "", "Read read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err) + return + } + if !bytes.Equal(buf1, buf) { + logError(testName, function, args, startTime, "", "Incorrect count of Read data", err) + return + } + + st, err := r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + + if st.Size != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) + return + } + + m, err = r.ReadAt(buf2, 512) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + if m != len(buf2) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err) + return + } + if !bytes.Equal(buf2, buf[512:1024]) { + logError(testName, function, args, startTime, "", "Incorrect count of ReadAt data", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + // Test Presigned Post Policy func testPresignedPostPolicy() { // initialize logging params @@ -10004,6 +10131,7 @@ func main() { testFPutObject() testGetObjectReadSeekFunctional() testGetObjectReadAtFunctional() + testGetObjectReadAtWhenEOFWasReached() testPresignedPostPolicy() testCopyObject() testComposeObjectErrorCases() From 6a767a3d6f42d59225296c52f6b243778327a9f5 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Tue, 9 Jul 2019 23:47:03 +0000 Subject: [PATCH 044/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 4be14ca51..2b562762b 100644 --- a/api.go +++ b/api.go @@ -103,7 +103,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.32" + libraryVersion = "v6.0.33" ) // User Agent should always following the below style. From a45c90c613339104297e73f4114c5c39927dd56e Mon Sep 17 00:00:00 2001 From: Kanagaraj M Date: Wed, 10 Jul 2019 22:34:39 +0530 Subject: [PATCH 045/215] fix error message for wrong content length on putobject (#1140) When putObject is called with bigger ContentLength than the actual object size, Unexpected EOF error is thrown which is not clear. This happens only with streaming signature. Now the SteamingReader Read will return a specific error message saying mismatch in content length, instead of Unexpected EOF. For example "http: ContentLength=75444 with Body length 70444" Fixes #1135 --- pkg/s3signer/request-signature-streaming.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/s3signer/request-signature-streaming.go b/pkg/s3signer/request-signature-streaming.go index c82e6bd37..810b47c4b 100644 --- a/pkg/s3signer/request-signature-streaming.go +++ b/pkg/s3signer/request-signature-streaming.go @@ -285,7 +285,7 @@ func (s *StreamingReader) Read(buf []byte) (int, error) { // bytes read from baseReader different than // content length provided. if s.bytesRead != s.contentLen { - return 0, io.ErrUnexpectedEOF + return 0, fmt.Errorf("http: ContentLength=%d with Body length %d", s.contentLen, s.bytesRead) } // Sign the chunk and write it to s.buf. From 3270fe6e579532e60e828a0a3eab1aa794eeba59 Mon Sep 17 00:00:00 2001 From: James Naftel Date: Wed, 10 Jul 2019 18:28:58 -0400 Subject: [PATCH 046/215] Example: Return MakeBucket error in FileUploader.go (#1143) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8a66bf5c9..bc0a10d8c 100644 --- a/README.md +++ b/README.md @@ -79,8 +79,8 @@ func main() { err = minioClient.MakeBucket(bucketName, location) if err != nil { // Check to see if we already own this bucket (which happens if you run this twice) - exists, err := minioClient.BucketExists(bucketName) - if err == nil && exists { + exists, errBucketExists := minioClient.BucketExists(bucketName) + if errBucketExists == nil && exists { log.Printf("We already own %s\n", bucketName) } else { log.Fatalln(err) From 753e5f73af12800035498a3efc6828e943143c5c Mon Sep 17 00:00:00 2001 From: Andreas Auernhammer Date: Thu, 11 Jul 2019 09:20:42 +0200 Subject: [PATCH 047/215] add `EndpointURL` method to S3 client (#1141) This commit adds an EndpointURL method to the S3 client such that calling code can get the S3 endpoint in all contexts. --- api.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api.go b/api.go index 2b562762b..acac3edb4 100644 --- a/api.go +++ b/api.go @@ -187,6 +187,12 @@ func NewWithOptions(endpoint string, opts *Options) (*Client, error) { return privateNew(endpoint, opts.Creds, opts.Secure, opts.Region, opts.BucketLookup) } +// EndpointURL returns the URL of the S3 endpoint. +func (c *Client) EndpointURL() *url.URL { + endpoint := *c.endpointURL // copy to prevent callers from modifying internal state + return &endpoint +} + // lockedRandSource provides protected rand source, implements rand.Source interface. type lockedRandSource struct { lk sync.Mutex From be06703be060e0c0c0505f31a3781b8908a32a00 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 16 Jul 2019 14:19:28 -0700 Subject: [PATCH 048/215] Remove port 9000 from play.min.io (#1144) --- README.md | 6 +++--- README_zh_CN.md | 6 +++--- docs/API.md | 2 +- docs/checker.go.template | 2 +- docs/zh_CN/API.md | 2 +- examples/minio/listenbucketnotification.go | 2 +- functional_tests.go | 4 ++-- pkg/credentials/config.json.sample | 2 +- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index bc0a10d8c..81320a14f 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ import ( ) func main() { - endpoint := "play.min.io:9000" + endpoint := "play.min.io" accessKeyID := "Q3AM3UQ867SPQQA43P2F" secretAccessKey := "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" useSSL := true @@ -49,7 +49,7 @@ func main() { ## Quick Start Example - File Uploader This example program connects to an object storage server, creates a bucket and uploads a file to the bucket. -We will use the MinIO server running at [https://play.min.io:9000](https://play.min.io:9000) in this example. Feel free to use this service for testing and development. Access credentials shown in this example are open to the public. +We will use the MinIO server running at [https://play.min.io](https://play.min.io) in this example. Feel free to use this service for testing and development. Access credentials shown in this example are open to the public. ### FileUploader.go ```go @@ -61,7 +61,7 @@ import ( ) func main() { - endpoint := "play.min.io:9000" + endpoint := "play.min.io" accessKeyID := "Q3AM3UQ867SPQQA43P2F" secretAccessKey := "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" useSSL := true diff --git a/README_zh_CN.md b/README_zh_CN.md index c9c5e3133..bc0c45c0f 100644 --- a/README_zh_CN.md +++ b/README_zh_CN.md @@ -43,7 +43,7 @@ import ( ) func main() { - endpoint := "play.min.io:9000" + endpoint := "play.min.io" accessKeyID := "Q3AM3UQ867SPQQA43P2F" secretAccessKey := "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" useSSL := true @@ -61,7 +61,7 @@ func main() { ## 示例-文件上传 本示例连接到一个对象存储服务,创建一个存储桶并上传一个文件到存储桶中。 -我们在本示例中使用运行在 [https://play.min.io:9000](https://play.min.io:9000) 上的MinIO服务,你可以用这个服务来开发和测试。示例中的访问凭据是公开的。 +我们在本示例中使用运行在 [https://play.min.io](https://play.min.io) 上的MinIO服务,你可以用这个服务来开发和测试。示例中的访问凭据是公开的。 ### FileUploader.go ```go @@ -73,7 +73,7 @@ import ( ) func main() { - endpoint := "play.min.io:9000" + endpoint := "play.min.io" accessKeyID := "Q3AM3UQ867SPQQA43P2F" secretAccessKey := "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" useSSL := true diff --git a/docs/API.md b/docs/API.md index 6e24ed91c..76e1a6215 100644 --- a/docs/API.md +++ b/docs/API.md @@ -18,7 +18,7 @@ func main() { ssl := true // Initialize minio client object. - minioClient, err := minio.New("play.min.io:9000", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", ssl) + minioClient, err := minio.New("play.min.io", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", ssl) if err != nil { fmt.Println(err) return diff --git a/docs/checker.go.template b/docs/checker.go.template index 750e1ec39..cf3c037f6 100644 --- a/docs/checker.go.template +++ b/docs/checker.go.template @@ -11,7 +11,7 @@ func main() { ssl := true // Initialize minio client object. - minioClient, err := minio.New("play.min.io:9000", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", ssl) + minioClient, err := minio.New("play.min.io", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", ssl) if err != nil { fmt.Println(err) return diff --git a/docs/zh_CN/API.md b/docs/zh_CN/API.md index a3778447a..1e478973c 100644 --- a/docs/zh_CN/API.md +++ b/docs/zh_CN/API.md @@ -18,7 +18,7 @@ func main() { ssl := true // 初使化minio client对象。 - minioClient, err := minio.New("play.min.io:9000", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", ssl) + minioClient, err := minio.New("play.min.io", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", ssl) if err != nil { fmt.Println(err) return diff --git a/examples/minio/listenbucketnotification.go b/examples/minio/listenbucketnotification.go index 3082f9f21..a899c1407 100644 --- a/examples/minio/listenbucketnotification.go +++ b/examples/minio/listenbucketnotification.go @@ -34,7 +34,7 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - minioClient, err := minio.New("play.min.io:9000", "YOUR-ACCESS", "YOUR-SECRET", true) + minioClient, err := minio.New("play.min.io", "YOUR-ACCESS", "YOUR-SECRET", true) if err != nil { log.Fatalln(err) } diff --git a/functional_tests.go b/functional_tests.go index fc8143f10..463552ce1 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -184,9 +184,9 @@ func isErrNotImplemented(err error) bool { func init() { // If server endpoint is not set, all tests default to - // using https://play.min.io:9000 + // using https://play.min.io if os.Getenv(serverEndpoint) == "" { - os.Setenv(serverEndpoint, "play.min.io:9000") + os.Setenv(serverEndpoint, "play.min.io") os.Setenv(accessKey, "Q3AM3UQ867SPQQA43P2F") os.Setenv(secretKey, "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG") os.Setenv(enableHTTPS, "1") diff --git a/pkg/credentials/config.json.sample b/pkg/credentials/config.json.sample index 0affa58cf..d793c9e0e 100644 --- a/pkg/credentials/config.json.sample +++ b/pkg/credentials/config.json.sample @@ -2,7 +2,7 @@ "version": "8", "hosts": { "play": { - "url": "https://play.min.io:9000", + "url": "https://play.min.io", "accessKey": "Q3AM3UQ867SPQQA43P2F", "secretKey": "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", "api": "S3v2" From 3ab0c3b54dab077c8b2cd6c58f04fafa4c529f1e Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Wed, 17 Jul 2019 23:03:00 +0000 Subject: [PATCH 049/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index acac3edb4..8e4021988 100644 --- a/api.go +++ b/api.go @@ -103,7 +103,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.33" + libraryVersion = "v6.0.34" ) // User Agent should always following the below style. From ae76eb2d7129530c3aba13720dd8edf492559efe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BA=84=E5=A4=A9=E7=BF=BC?= Date: Tue, 13 Aug 2019 09:39:01 +0800 Subject: [PATCH 050/215] add WithContext API in Core (#1148) --- core.go | 71 +++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 59 insertions(+), 12 deletions(-) diff --git a/core.go b/core.go index ba574c44e..4f5719343 100644 --- a/core.go +++ b/core.go @@ -55,9 +55,23 @@ func (c Core) ListObjectsV2(bucketName, objectPrefix, continuationToken string, return c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, maxkeys, startAfter) } +// CopyObjectWithContext - copies an object from source object to destination object on server side. +func (c Core) CopyObjectWithContext(ctx context.Context, sourceBucket, sourceObject, destBucket, destObject string, metadata map[string]string) (ObjectInfo, error) { + return c.copyObjectDo(ctx, sourceBucket, sourceObject, destBucket, destObject, metadata) +} + // CopyObject - copies an object from source object to destination object on server side. func (c Core) CopyObject(sourceBucket, sourceObject, destBucket, destObject string, metadata map[string]string) (ObjectInfo, error) { - return c.copyObjectDo(context.Background(), sourceBucket, sourceObject, destBucket, destObject, metadata) + return c.CopyObjectWithContext(context.Background(), sourceBucket, sourceObject, destBucket, destObject, metadata) +} + +// CopyObjectPartWithContext - creates a part in a multipart upload by copying (a +// part of) an existing object. +func (c Core) CopyObjectPartWithContext(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, + partID int, startOffset, length int64, metadata map[string]string) (p CompletePart, err error) { + + return c.copyObjectPartDo(ctx, srcBucket, srcObject, destBucket, destObject, uploadID, + partID, startOffset, length, metadata) } // CopyObjectPart - creates a part in a multipart upload by copying (a @@ -65,12 +79,12 @@ func (c Core) CopyObject(sourceBucket, sourceObject, destBucket, destObject stri func (c Core) CopyObjectPart(srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset, length int64, metadata map[string]string) (p CompletePart, err error) { - return c.copyObjectPartDo(context.Background(), srcBucket, srcObject, destBucket, destObject, uploadID, + return c.CopyObjectPartWithContext(context.Background(), srcBucket, srcObject, destBucket, destObject, uploadID, partID, startOffset, length, metadata) } -// PutObject - Upload object. Uploads using single PUT call. -func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, metadata map[string]string, sse encrypt.ServerSide) (ObjectInfo, error) { +// PutObjectWithContext - Upload object. Uploads using single PUT call. +func (c Core) PutObjectWithContext(ctx context.Context, bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, metadata map[string]string, sse encrypt.ServerSide) (ObjectInfo, error) { opts := PutObjectOptions{} m := make(map[string]string) for k, v := range metadata { @@ -92,7 +106,12 @@ func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Ba } opts.UserMetadata = m opts.ServerSideEncryption = sse - return c.putObjectDo(context.Background(), bucket, object, data, md5Base64, sha256Hex, size, opts) + return c.putObjectDo(ctx, bucket, object, data, md5Base64, sha256Hex, size, opts) +} + +// PutObject - Upload object. Uploads using single PUT call. +func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, metadata map[string]string, sse encrypt.ServerSide) (ObjectInfo, error) { + return c.PutObjectWithContext(context.Background(), bucket, object, data, size, md5Base64, sha256Hex, metadata, sse) } // NewMultipartUpload - Initiates new multipart upload and returns the new uploadID. @@ -106,9 +125,14 @@ func (c Core) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, de return c.listMultipartUploadsQuery(bucket, keyMarker, uploadIDMarker, prefix, delimiter, maxUploads) } +// PutObjectPartWithContext - Upload an object part. +func (c Core) PutObjectPartWithContext(ctx context.Context, bucket, object, uploadID string, partID int, data io.Reader, size int64, md5Base64, sha256Hex string, sse encrypt.ServerSide) (ObjectPart, error) { + return c.uploadPart(ctx, bucket, object, uploadID, data, partID, md5Base64, sha256Hex, size, sse) +} + // PutObjectPart - Upload an object part. func (c Core) PutObjectPart(bucket, object, uploadID string, partID int, data io.Reader, size int64, md5Base64, sha256Hex string, sse encrypt.ServerSide) (ObjectPart, error) { - return c.uploadPart(context.Background(), bucket, object, uploadID, data, partID, md5Base64, sha256Hex, size, sse) + return c.PutObjectPartWithContext(context.Background(), bucket, object, uploadID, partID, data, size, md5Base64, sha256Hex, sse) } // ListObjectParts - List uploaded parts of an incomplete upload.x @@ -116,17 +140,27 @@ func (c Core) ListObjectParts(bucket, object, uploadID string, partNumberMarker return c.listObjectPartsQuery(bucket, object, uploadID, partNumberMarker, maxParts) } -// CompleteMultipartUpload - Concatenate uploaded parts and commit to an object. -func (c Core) CompleteMultipartUpload(bucket, object, uploadID string, parts []CompletePart) (string, error) { - res, err := c.completeMultipartUpload(context.Background(), bucket, object, uploadID, completeMultipartUpload{ +// CompleteMultipartUploadWithContext - Concatenate uploaded parts and commit to an object. +func (c Core) CompleteMultipartUploadWithContext(ctx context.Context, bucket, object, uploadID string, parts []CompletePart) (string, error) { + res, err := c.completeMultipartUpload(ctx, bucket, object, uploadID, completeMultipartUpload{ Parts: parts, }) return res.ETag, err } +// CompleteMultipartUpload - Concatenate uploaded parts and commit to an object. +func (c Core) CompleteMultipartUpload(bucket, object, uploadID string, parts []CompletePart) (string, error) { + return c.CompleteMultipartUploadWithContext(context.Background(), bucket, object, uploadID, parts) +} + +// AbortMultipartUploadWithContext - Abort an incomplete upload. +func (c Core) AbortMultipartUploadWithContext(ctx context.Context, bucket, object, uploadID string) error { + return c.abortMultipartUpload(ctx, bucket, object, uploadID) +} + // AbortMultipartUpload - Abort an incomplete upload. func (c Core) AbortMultipartUpload(bucket, object, uploadID string) error { - return c.abortMultipartUpload(context.Background(), bucket, object, uploadID) + return c.AbortMultipartUploadWithContext(context.Background(), bucket, object, uploadID) } // GetBucketPolicy - fetches bucket access policy for a given bucket. @@ -139,15 +173,28 @@ func (c Core) PutBucketPolicy(bucket, bucketPolicy string) error { return c.putBucketPolicy(bucket, bucketPolicy) } +// GetObjectWithContext is a lower level API implemented to support reading +// partial objects and also downloading objects with special conditions +// matching etag, modtime etc. +func (c Core) GetObjectWithContext(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, error) { + return c.getObject(ctx, bucketName, objectName, opts) +} + // GetObject is a lower level API implemented to support reading // partial objects and also downloading objects with special conditions // matching etag, modtime etc. func (c Core) GetObject(bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, error) { - return c.getObject(context.Background(), bucketName, objectName, opts) + return c.GetObjectWithContext(context.Background(), bucketName, objectName, opts) +} + +// StatObjectWithContext is a lower level API implemented to support special +// conditions matching etag, modtime on a request. +func (c Core) StatObjectWithContext(ctx context.Context, bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { + return c.statObject(ctx, bucketName, objectName, opts) } // StatObject is a lower level API implemented to support special // conditions matching etag, modtime on a request. func (c Core) StatObject(bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { - return c.statObject(context.Background(), bucketName, objectName, opts) + return c.StatObjectWithContext(context.Background(), bucketName, objectName, opts) } From cf0a2bd391ee566dbdc00b048241fd5c77f8959d Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Wed, 14 Aug 2019 18:12:32 +0000 Subject: [PATCH 051/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 8e4021988..772c09b28 100644 --- a/api.go +++ b/api.go @@ -103,7 +103,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.34" + libraryVersion = "v6.0.35" ) // User Agent should always following the below style. From 9009302867770db7167735f410c9235796ddc691 Mon Sep 17 00:00:00 2001 From: poornas Date: Thu, 15 Aug 2019 18:29:03 -0400 Subject: [PATCH 052/215] Fix broken link to go install docs (#1149) --- README.md | 2 +- README_zh_CN.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 81320a14f..73818001d 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ The MinIO Go Client SDK provides simple APIs to access any Amazon S3 compatible This quickstart guide will show you how to install the MinIO client SDK, connect to MinIO, and provide a walkthrough for a simple file uploader. For a complete list of APIs and examples, please take a look at the [Go Client API Reference](https://docs.min.io/docs/golang-client-api-reference). -This document assumes that you have a working [Go development environment](https://docs.min.io/docs/how-to-install-golang). +This document assumes that you have a working [Go development environment](https://golang.org/doc/install). ## Download from Github ```sh diff --git a/README_zh_CN.md b/README_zh_CN.md index bc0c45c0f..b5ba0c1d0 100644 --- a/README_zh_CN.md +++ b/README_zh_CN.md @@ -16,7 +16,7 @@ MinIO Go Client SDK提供了简单的API来访问任何与Amazon S3兼容的对 本文我们将学习如何安装MinIO client SDK,连接到MinIO,并提供一下文件上传的示例。对于完整的API以及示例,请参考[Go Client API Reference](https://docs.min.io/docs/golang-client-api-reference)。 -本文假设你已经有 [Go开发环境](https://docs.min.io/docs/how-to-install-golang)。 +本文假设你已经有 [Go开发环境](https://golang.org/doc/install)。 ## 从Github下载 ```sh From 29666b6856f41b22dd60af2503901f3943c7306f Mon Sep 17 00:00:00 2001 From: Aditya Manthramurthy Date: Thu, 29 Aug 2019 18:43:48 -0700 Subject: [PATCH 053/215] Add LDAP client integration (#1152) --- pkg/credentials/sts_ldap_identity.go | 119 +++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 pkg/credentials/sts_ldap_identity.go diff --git a/pkg/credentials/sts_ldap_identity.go b/pkg/credentials/sts_ldap_identity.go new file mode 100644 index 000000000..875cf1064 --- /dev/null +++ b/pkg/credentials/sts_ldap_identity.go @@ -0,0 +1,119 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "encoding/xml" + "errors" + "net/http" + "net/url" + "time" +) + +// AssumeRoleWithLDAPResponse contains the result of successful +// AssumeRoleWithLDAPIdentity request +type AssumeRoleWithLDAPResponse struct { + XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleWithClientGrantsResponse" json:"-"` + Result LDAPIdentityResult `xml:"AssumeRoleWithLDAPIdentity"` + ResponseMetadata struct { + RequestID string `xml:"RequestId,omitempty"` + } `xml:"ResponseMetadata,omitempty"` +} + +// LDAPIdentityResult - contains credentials for a successful +// AssumeRoleWithLDAPIdentity request. +type LDAPIdentityResult struct { + Credentials struct { + AccessKey string `xml:"AccessKeyId" json:"accessKey,omitempty"` + SecretKey string `xml:"SecretAccessKey" json:"secretKey,omitempty"` + Expiration time.Time `xml:"Expiration" json:"expiration,omitempty"` + SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty"` + } `xml:",omitempty"` + + SubjectFromToken string `xml:",omitempty"` +} + +// LDAPIdentity retrieves credentials from MinIO +type LDAPIdentity struct { + Expiry + + stsEndpoint string + + ldapUsername, ldapPassword string +} + +// NewLDAPIdentity returns new credentials object that uses LDAP +// Identity. +func NewLDAPIdentity(stsEndpoint, ldapUsername, ldapPassword string) (*Credentials, error) { + return New(&LDAPIdentity{ + stsEndpoint: stsEndpoint, + ldapUsername: ldapUsername, + ldapPassword: ldapPassword, + }), nil +} + +// Retrieve gets the credential by calling the MinIO STS API for +// LDAP on the configured stsEndpoint. +func (k *LDAPIdentity) Retrieve() (value Value, err error) { + u, kerr := url.Parse(k.stsEndpoint) + if kerr != nil { + err = kerr + return + } + + clnt := &http.Client{Transport: http.DefaultTransport} + v := url.Values{} + v.Set("Action", "AssumeRoleWithLDAPIdentity") + v.Set("Version", "2011-06-15") + v.Set("LDAPUsername", k.ldapUsername) + v.Set("LDAPPassword", k.ldapPassword) + + u.RawQuery = v.Encode() + + req, kerr := http.NewRequest("POST", u.String(), nil) + if kerr != nil { + err = kerr + return + } + + resp, kerr := clnt.Do(req) + if kerr != nil { + err = kerr + return + } + + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + err = errors.New(resp.Status) + return + } + + r := AssumeRoleWithLDAPResponse{} + if err = xml.NewDecoder(resp.Body).Decode(&r); err != nil { + return + } + + cr := r.Result.Credentials + k.SetExpiration(cr.Expiration, DefaultExpiryWindow) + return Value{ + AccessKeyID: cr.AccessKey, + SecretAccessKey: cr.SecretKey, + SessionToken: cr.SessionToken, + SignerType: SignatureV4, + }, nil +} From 8afcf7be72ae6d4aa17ad3b5449b874df2d92003 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 2 Sep 2019 22:17:38 -0700 Subject: [PATCH 054/215] Fix documentation on minio-go download (#1155) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 73818001d..810a93941 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This document assumes that you have a working [Go development environment](https ## Download from Github ```sh -go get -u github.com/minio/minio-go +GO111MODULE=on go get github.com/minio/minio-go/v6 ``` ## Initialize MinIO Client From f0765917de41346de96e27fc09711a25e9d3455d Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Wed, 11 Sep 2019 18:48:14 +0000 Subject: [PATCH 055/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 772c09b28..e1dccc6d5 100644 --- a/api.go +++ b/api.go @@ -103,7 +103,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.35" + libraryVersion = "v6.0.36" ) // User Agent should always following the below style. From 72eb8ecfaaafa8bfaeb0e99590ef0a1161f714d5 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 16 Sep 2019 11:23:05 -0700 Subject: [PATCH 056/215] Bump builds to 1.13 only (#1158) --- .travis.yml | 6 +++--- api.go | 3 ++- go.mod | 7 ++----- go.sum | 23 ++--------------------- pkg/s3signer/utils.go | 3 ++- utils.go | 3 ++- 6 files changed, 13 insertions(+), 32 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5dbda5e7d..6400cc528 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ env: - ARCH=x86_64 go: -- 1.12.x +- 1.13.x - tip matrix: @@ -18,10 +18,10 @@ matrix: before_install: - sudo apt-get install devscripts - - curl -O https://dl.minio.io/server/minio/release/linux-amd64/minio && chmod +x ./minio + - go get github.com/minio/minio - sudo cp testcerts/public.crt /usr/local/share/ca-certificates/ - sudo update-ca-certificates - - MINIO_ACCESS_KEY=minio MINIO_SECRET_KEY=minio123 ./minio server --compat --quiet --certs-dir testcerts data 2>&1 > minio.log & + - MINIO_ACCESS_KEY=minio MINIO_SECRET_KEY=minio123 ${GOPATH}/bin/minio server --compat --quiet --certs-dir testcerts data 2>&1 > minio.log & script: - diff -au <(gofmt -d .) <(printf "") diff --git a/api.go b/api.go index e1dccc6d5..3eed21e52 100644 --- a/api.go +++ b/api.go @@ -21,7 +21,6 @@ import ( "bytes" "context" "crypto/md5" - "crypto/sha256" "errors" "fmt" "hash" @@ -39,6 +38,8 @@ import ( "sync" "time" + "github.com/minio/sha256-simd" + "golang.org/x/net/publicsuffix" "github.com/minio/minio-go/v6/pkg/credentials" diff --git a/go.mod b/go.mod index 9d342468d..9e5334f11 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,11 @@ module github.com/minio/minio-go/v6 go 1.12 require ( - github.com/a8m/mark v0.1.1-0.20170507133748-44f2db618845 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect - github.com/gernest/wow v0.1.0 // indirect - github.com/minio/cli v1.20.0 // indirect + github.com/minio/sha256-simd v0.1.0 github.com/mitchellh/go-homedir v1.1.0 - github.com/sirupsen/logrus v1.4.2 // indirect github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f golang.org/x/net v0.0.0-20190522155817-f3200d17e092 + golang.org/x/sys v0.0.0-20190422165155-953cdadca894 // indirect gopkg.in/ini.v1 v1.42.0 ) diff --git a/go.sum b/go.sum index 62a4f31fa..168fdd9ff 100644 --- a/go.sum +++ b/go.sum @@ -1,30 +1,15 @@ -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/a8m/mark v0.1.1-0.20170507133748-44f2db618845 h1:hIjQrEARcc9LcH8igte3JBpWBZ7+SpinU70dOjU/afo= -github.com/a8m/mark v0.1.1-0.20170507133748-44f2db618845/go.mod h1:c8Mh99Cw82nrsAnPgxQSZHkswVOJF7/MqZb1ZdvriLM= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/gernest/wow v0.1.0 h1:g9xdwCwP0+xgVYlA2sopI0gZHqXe7HjI/7/LykG4fks= -github.com/gernest/wow v0.1.0/go.mod h1:dEPabJRi5BneI1Nev1VWo0ZlcTWibHWp43qxKms4elY= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/minio/cli v1.20.0 h1:OVNIt8Rg5+mpYb8siWT2gBV5hvUyFbRvBikC+Ytvf5A= -github.com/minio/cli v1.20.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY= +github.com/minio/sha256-simd v0.1.0 h1:U41/2erhAKcmSI14xh/ZTUdBPOzDOIfS93ibzUSl8KM= +github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -32,7 +17,6 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/sys v0.0.0-20190116161447-11f53e031339/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -41,8 +25,5 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/s3signer/utils.go b/pkg/s3signer/utils.go index d07ff6dec..934e33a48 100644 --- a/pkg/s3signer/utils.go +++ b/pkg/s3signer/utils.go @@ -19,9 +19,10 @@ package s3signer import ( "crypto/hmac" - "crypto/sha256" "net/http" "strings" + + "github.com/minio/sha256-simd" ) // unsignedPayload - value to be set to X-Amz-Content-Sha256 header when diff --git a/utils.go b/utils.go index fc30c1ab7..2a743a834 100644 --- a/utils.go +++ b/utils.go @@ -19,7 +19,6 @@ package minio import ( "crypto/md5" - "crypto/sha256" "encoding/base64" "encoding/hex" "encoding/xml" @@ -32,6 +31,8 @@ import ( "strings" "time" + "github.com/minio/sha256-simd" + "github.com/minio/minio-go/v6/pkg/s3utils" ) From 8d3ebf1e92506f8fea2760b4716e494f328ba2b3 Mon Sep 17 00:00:00 2001 From: poornas Date: Tue, 17 Sep 2019 10:52:55 -0700 Subject: [PATCH 057/215] Fix functional test (#1160) Log CopyObject error in testStorageClassMetadataCopyObject - fixes https://github.com/minio/minio/issues/8165 --- functional_tests.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/functional_tests.go b/functional_tests.go index 463552ce1..8994395f5 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -8566,6 +8566,8 @@ func testStorageClassMetadataCopyObject() { fetchMeta := func(object string) (h http.Header) { objInfo, err := c.StatObject(bucketName, object, minio.StatObjectOptions{}) + args["bucket"] = bucketName + args["object"] = object if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return @@ -8600,7 +8602,9 @@ func testStorageClassMetadataCopyObject() { // Make server side copy of object uploaded in previous step src := minio.NewSourceInfo(bucketName, "srcObjectRRSClass", nil) dst, err := minio.NewDestinationInfo(bucketName, "srcObjectRRSClassCopy", nil, nil) - c.CopyObject(dst, src) + if err = c.CopyObject(dst, src); err != nil { + logError(testName,function,args,startTime,"", "CopyObject failed on RRS", err) + } // Get the returned metadata returnedMeta := fetchMeta("srcObjectRRSClassCopy") @@ -8625,8 +8629,9 @@ func testStorageClassMetadataCopyObject() { // Make server side copy of object uploaded in previous step src = minio.NewSourceInfo(bucketName, "srcObjectSSClass", nil) dst, err = minio.NewDestinationInfo(bucketName, "srcObjectSSClassCopy", nil, nil) - c.CopyObject(dst, src) - + if err = c.CopyObject(dst, src); err != nil { + logError(testName, function, args, startTime, "", "CopyObject failed on SS", err) + } // Fetch the meta data of copied object if reflect.DeepEqual(metadata, fetchMeta("srcObjectSSClassCopy")) { logError(testName, function, args, startTime, "", "Metadata verification failed, STANDARD storage class should not be a part of response metadata", err) From 1e017194bd0a9f77db34525b4592e73f29ced561 Mon Sep 17 00:00:00 2001 From: kannappanr <30541348+kannappanr@users.noreply.github.com> Date: Thu, 19 Sep 2019 03:35:09 +0530 Subject: [PATCH 058/215] set encoding type to url in listobjects API (#1161) Also fix formatting in functional tests --- api-list.go | 6 ++++++ functional_tests.go | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/api-list.go b/api-list.go index b9c0f5d8d..c7368fcbf 100644 --- a/api-list.go +++ b/api-list.go @@ -192,6 +192,9 @@ func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken s // Always set list-type in ListObjects V2 urlValues.Set("list-type", "2") + // Always set encoding-type in ListObjects V2 + urlValues.Set("encoding-type", "url") + // Set object prefix, prefix value to be set to empty is okay. urlValues.Set("prefix", objectPrefix) @@ -397,6 +400,9 @@ func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimit // Set max keys. urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys)) + // Always set encoding-type + urlValues.Set("encoding-type", "url") + // Execute GET on bucket to list objects. resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ bucketName: bucketName, diff --git a/functional_tests.go b/functional_tests.go index 8994395f5..30199cfe0 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -8603,7 +8603,7 @@ func testStorageClassMetadataCopyObject() { src := minio.NewSourceInfo(bucketName, "srcObjectRRSClass", nil) dst, err := minio.NewDestinationInfo(bucketName, "srcObjectRRSClassCopy", nil, nil) if err = c.CopyObject(dst, src); err != nil { - logError(testName,function,args,startTime,"", "CopyObject failed on RRS", err) + logError(testName, function, args, startTime, "", "CopyObject failed on RRS", err) } // Get the returned metadata From eed067a2ae74f7c82c64f379f4fd38538261f3e2 Mon Sep 17 00:00:00 2001 From: kannappanr <30541348+kannappanr@users.noreply.github.com> Date: Thu, 19 Sep 2019 06:18:06 +0530 Subject: [PATCH 059/215] Remove validator.go inside docs (#1162) Existing code cannot be used as is. Needs to be rewritten. OK to remove it now. --- Makefile | 5 +- docs/checker.go.template | 21 ---- docs/validator.go | 227 --------------------------------------- 3 files changed, 1 insertion(+), 252 deletions(-) delete mode 100644 docs/checker.go.template delete mode 100644 docs/validator.go diff --git a/Makefile b/Makefile index 892f8cf50..5cfe33a46 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ all: checks .PHONY: examples docs -checks: vet test examples docs functional-test +checks: vet test examples functional-test vet: @GO111MODULE=on go vet ./... @@ -13,8 +13,5 @@ test: examples: @mkdir -p /tmp/examples && for i in $(echo examples/s3/*); do go build -o /tmp/examples/$(basename ${i:0:-3}) ${i}; done -docs: - @(cd docs; GO111MODULE=on go build validator.go && ./validator -m ../docs/API.md -t checker.go.tpl) - functional-test: @GO111MODULE=on SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minio SECRET_KEY=minio123 ENABLE_HTTPS=1 MINT_MODE=full go run functional_tests.go diff --git a/docs/checker.go.template b/docs/checker.go.template deleted file mode 100644 index cf3c037f6..000000000 --- a/docs/checker.go.template +++ /dev/null @@ -1,21 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/minio/minio-go/v6" -) - -func main() { - // Use a secure connection. - ssl := true - - // Initialize minio client object. - minioClient, err := minio.New("play.min.io", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", ssl) - if err != nil { - fmt.Println(err) - return - } - - {{.Text}} -} diff --git a/docs/validator.go b/docs/validator.go deleted file mode 100644 index e75de75bd..000000000 --- a/docs/validator.go +++ /dev/null @@ -1,227 +0,0 @@ -// +build ignore - -/* - * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "fmt" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "strings" - "text/template" - - "github.com/a8m/mark" - "github.com/gernest/wow" - "github.com/gernest/wow/spin" - "github.com/minio/cli" -) - -func init() { - // Validate go binary. - if _, err := exec.LookPath("go"); err != nil { - panic(err) - } -} - -var globalFlags = []cli.Flag{ - cli.StringFlag{ - Name: "m", - Value: "API.md", - Usage: "Path to markdown api documentation.", - }, - cli.StringFlag{ - Name: "t", - Value: "checker.go.template", - Usage: "Template used for generating the programs.", - }, - cli.IntFlag{ - Name: "skip", - Value: 2, - Usage: "Skip entries before validating the code.", - }, -} - -func runGofmt(path string) (msg string, err error) { - cmdArgs := []string{"-s", "-w", "-l", path} - cmd := exec.Command("gofmt", cmdArgs...) - stdoutStderr, err := cmd.CombinedOutput() - if err != nil { - return "", err - } - return string(stdoutStderr), nil -} - -func runGoImports(path string) (msg string, err error) { - cmdArgs := []string{"-w", path} - cmd := exec.Command("goimports", cmdArgs...) - stdoutStderr, err := cmd.CombinedOutput() - if err != nil { - return string(stdoutStderr), err - } - return string(stdoutStderr), nil -} - -func runGoBuild(path string) (msg string, err error) { - // Go build the path. - cmdArgs := []string{"build", "-o", "/dev/null", path} - cmd := exec.Command("go", cmdArgs...) - stdoutStderr, err := cmd.CombinedOutput() - if err != nil { - return string(stdoutStderr), err - } - return string(stdoutStderr), nil -} - -func validatorAction(ctx *cli.Context) error { - if !ctx.IsSet("m") || !ctx.IsSet("t") { - return nil - } - docPath := ctx.String("m") - var err error - docPath, err = filepath.Abs(docPath) - if err != nil { - return err - } - data, err := ioutil.ReadFile(docPath) - if err != nil { - return err - } - - templatePath := ctx.String("t") - templatePath, err = filepath.Abs(templatePath) - if err != nil { - return err - } - - skipEntries := ctx.Int("skip") - m := mark.New(string(data), &mark.Options{ - Gfm: true, // Github markdown support is enabled by default. - }) - - t, err := template.ParseFiles(templatePath) - if err != nil { - return err - } - - tmpDir, err := ioutil.TempDir("", "md-verifier") - if err != nil { - return err - } - defer os.RemoveAll(tmpDir) - - entryN := 1 - for i := mark.NodeText; i < mark.NodeCheckbox; i++ { - if mark.NodeCode != mark.NodeType(i) { - m.AddRenderFn(mark.NodeType(i), func(node mark.Node) (s string) { - return "" - }) - continue - } - m.AddRenderFn(mark.NodeCode, func(node mark.Node) (s string) { - p, ok := node.(*mark.CodeNode) - if !ok { - return - } - p.Text = strings.NewReplacer("<", "<", ">", ">", """, `"`, "&", "&").Replace(p.Text) - if skipEntries > 0 { - skipEntries-- - return - } - - testFilePath := filepath.Join(tmpDir, "example.go") - w, werr := os.Create(testFilePath) - if werr != nil { - panic(werr) - } - t.Execute(w, p) - w.Sync() - w.Close() - entryN++ - - msg, err := runGofmt(testFilePath) - if err != nil { - fmt.Printf("Failed running gofmt on %s, with (%s):(%s)\n", testFilePath, msg, err) - os.Exit(-1) - } - - msg, err = runGoImports(testFilePath) - if err != nil { - fmt.Printf("Failed running gofmt on %s, with (%s):(%s)\n", testFilePath, msg, err) - os.Exit(-1) - } - - msg, err = runGoBuild(testFilePath) - if err != nil { - fmt.Printf("Failed running gobuild on %s, with (%s):(%s)\n", testFilePath, msg, err) - fmt.Printf("Code with possible issue in %s:\n%s", docPath, p.Text) - fmt.Printf("To test `go build %s`\n", testFilePath) - os.Exit(-1) - } - - // Once successfully built remove the test file - os.Remove(testFilePath) - return - }) - } - - w := wow.New(os.Stdout, spin.Get(spin.Moon), fmt.Sprintf(" Running validation tests in %s", tmpDir)) - - w.Start() - // Render markdown executes our checker on each code blocks. - _ = m.Render() - w.PersistWith(spin.Get(spin.Runner), " Successfully finished tests") - w.Stop() - - return nil -} - -func main() { - app := cli.NewApp() - app.Action = validatorAction - app.HideVersion = true - app.HideHelpCommand = true - app.Usage = "Validates code block sections inside API.md" - app.Author = "min.io" - app.Flags = globalFlags - // Help template for validator - app.CustomAppHelpTemplate = `NAME: - {{.Name}} - {{.Usage}} - -USAGE: - {{.Name}} {{if .VisibleFlags}}[FLAGS] {{end}}COMMAND{{if .VisibleFlags}} [COMMAND FLAGS | -h]{{end}} [ARGUMENTS...] - -COMMANDS: - {{range .VisibleCommands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}} - {{end}}{{if .VisibleFlags}} -FLAGS: - {{range .VisibleFlags}}{{.}} - {{end}}{{end}} -TEMPLATE: - Validator uses Go's 'text/template' formatting so you need to ensure - your template is formatted correctly, check 'docs/checker.go.template' - -USAGE: - go run docs/validator.go -m docs/API.md -t /tmp/mycode.go.template - -` - app.Run(os.Args) - -} From ba491b60f605086f19b4c2c60c0683755419ec7b Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Thu, 19 Sep 2019 01:04:37 +0000 Subject: [PATCH 060/215] Update version to next release --- api.go | 2 +- go.mod | 3 ++- go.sum | 9 +++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/api.go b/api.go index 3eed21e52..2a51d3286 100644 --- a/api.go +++ b/api.go @@ -104,7 +104,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.36" + libraryVersion = "v6.0.37" ) // User Agent should always following the below style. diff --git a/go.mod b/go.mod index 9e5334f11..d728ac7b8 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,12 @@ module github.com/minio/minio-go/v6 go 1.12 require ( + github.com/dustin/go-humanize v1.0.0 // indirect github.com/minio/sha256-simd v0.1.0 github.com/mitchellh/go-homedir v1.1.0 + github.com/sirupsen/logrus v1.4.2 // indirect github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f golang.org/x/net v0.0.0-20190522155817-f3200d17e092 - golang.org/x/sys v0.0.0-20190422165155-953cdadca894 // indirect gopkg.in/ini.v1 v1.42.0 ) diff --git a/go.sum b/go.sum index 168fdd9ff..a8ad8bde4 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,24 @@ +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/minio/sha256-simd v0.1.0 h1:U41/2erhAKcmSI14xh/ZTUdBPOzDOIfS93ibzUSl8KM= github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= From 89d5404ffa4a81b3e719a335352edf902b0efa95 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 23 Sep 2019 13:05:38 -0700 Subject: [PATCH 061/215] Fix listObjects hang when listing special characters (#1165) This PR 1e017194bd0a9f77db34525b4592e73f29ced561 introduced a regression in handling special character file listing, this causes `mc ls` like commands to hang. Fixes https://github.com/minio/mc/issues/2903 --- api-list.go | 85 ++++++++++++++++++++++++++++++++++++++++++++++------- go.mod | 2 +- go.sum | 4 +-- 3 files changed, 78 insertions(+), 13 deletions(-) diff --git a/api-list.go b/api-list.go index c7368fcbf..0c6a4681f 100644 --- a/api-list.go +++ b/api-list.go @@ -128,6 +128,13 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d // If contents are available loop through and send over channel. for _, object := range result.Contents { + object.Key, err = url.QueryUnescape(object.Key) + if err != nil { + objectStatCh <- ObjectInfo{ + Err: err, + } + return + } select { // Send object content. case objectStatCh <- object: @@ -140,12 +147,17 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d // Send all common prefixes if any. // NOTE: prefixes are only present if the request is delimited. for _, obj := range result.CommonPrefixes { + objInfo := ObjectInfo{} + objInfo.Key, err = url.QueryUnescape(obj.Prefix) + if err != nil { + objectStatCh <- ObjectInfo{ + Err: err, + } + return + } select { // Send object prefixes. - case objectStatCh <- ObjectInfo{ - Key: obj.Prefix, - Size: 0, - }: + case objectStatCh <- objInfo: // If receives done from the caller, return here. case <-doneCh: return @@ -154,7 +166,13 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d // If continuation token present, save it for next request. if result.NextContinuationToken != "" { - continuationToken = result.NextContinuationToken + continuationToken, err = url.QueryUnescape(result.NextContinuationToken) + if err != nil { + objectStatCh <- ObjectInfo{ + Err: err, + } + return + } } // Listing ends result is not truncated, return right here. @@ -320,6 +338,13 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don // If contents are available loop through and send over channel. for _, object := range result.Contents { + object.Key, err = url.QueryUnescape(object.Key) + if err != nil { + objectStatCh <- ObjectInfo{ + Err: err, + } + return + } // Save the marker. marker = object.Key select { @@ -335,7 +360,13 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don // NOTE: prefixes are only present if the request is delimited. for _, obj := range result.CommonPrefixes { object := ObjectInfo{} - object.Key = obj.Prefix + object.Key, err = url.QueryUnescape(obj.Prefix) + if err != nil { + objectStatCh <- ObjectInfo{ + Err: err, + } + return + } object.Size = 0 select { // Send object prefixes. @@ -348,7 +379,13 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don // If next marker present, save it for next request. if result.NextMarker != "" { - marker = result.NextMarker + marker, err = url.QueryUnescape(result.NextMarker) + if err != nil { + objectStatCh <- ObjectInfo{ + Err: err, + } + return + } } // Listing ends result is not truncated, return right here. @@ -495,8 +532,20 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive return } // Save objectMarker and uploadIDMarker for next request. - objectMarker = result.NextKeyMarker - uploadIDMarker = result.NextUploadIDMarker + objectMarker, err = url.QueryUnescape(result.NextKeyMarker) + if err != nil { + objectMultipartStatCh <- ObjectMultipartInfo{ + Err: err, + } + return + } + uploadIDMarker, err = url.QueryUnescape(result.NextUploadIDMarker) + if err != nil { + objectMultipartStatCh <- ObjectMultipartInfo{ + Err: err, + } + return + } // Send all multipart uploads. for _, obj := range result.Uploads { // Calculate total size of the uploaded parts if 'aggregateSize' is enabled. @@ -510,6 +559,13 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive continue } } + obj.Key, err = url.QueryUnescape(obj.Key) + if err != nil { + objectMultipartStatCh <- ObjectMultipartInfo{ + Err: err, + } + return + } select { // Send individual uploads here. case objectMultipartStatCh <- obj: @@ -522,7 +578,13 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive // NOTE: prefixes are only present if the request is delimited. for _, obj := range result.CommonPrefixes { object := ObjectMultipartInfo{} - object.Key = obj.Prefix + object.Key, err = url.QueryUnescape(obj.Prefix) + if err != nil { + objectMultipartStatCh <- ObjectMultipartInfo{ + Err: err, + } + return + } object.Size = 0 select { // Send delimited prefixes here. @@ -573,6 +635,9 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker, // Set delimiter, delimiter value to be set to empty is okay. urlValues.Set("delimiter", delimiter) + // Always set encoding-type + urlValues.Set("encoding-type", "url") + // maxUploads should be 1000 or less. if maxUploads == 0 || maxUploads > 1000 { maxUploads = 1000 diff --git a/go.mod b/go.mod index d728ac7b8..2dc189d1a 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.12 require ( github.com/dustin/go-humanize v1.0.0 // indirect - github.com/minio/sha256-simd v0.1.0 + github.com/minio/sha256-simd v0.1.1 github.com/mitchellh/go-homedir v1.1.0 github.com/sirupsen/logrus v1.4.2 // indirect github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect diff --git a/go.sum b/go.sum index a8ad8bde4..b3cacbda4 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/minio/sha256-simd v0.1.0 h1:U41/2erhAKcmSI14xh/ZTUdBPOzDOIfS93ibzUSl8KM= -github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= From 1446126c7143af2e321ee7eb329b946c01d8541e Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 23 Sep 2019 15:23:23 -0700 Subject: [PATCH 062/215] Change LDAP XML names to follow server changes (#1164) Depends on https://github.com/minio/minio/pull/8285 --- pkg/credentials/sts_ldap_identity.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/credentials/sts_ldap_identity.go b/pkg/credentials/sts_ldap_identity.go index 875cf1064..b72ac061c 100644 --- a/pkg/credentials/sts_ldap_identity.go +++ b/pkg/credentials/sts_ldap_identity.go @@ -28,8 +28,8 @@ import ( // AssumeRoleWithLDAPResponse contains the result of successful // AssumeRoleWithLDAPIdentity request type AssumeRoleWithLDAPResponse struct { - XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleWithClientGrantsResponse" json:"-"` - Result LDAPIdentityResult `xml:"AssumeRoleWithLDAPIdentity"` + XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleWithLDAPIdentityResponse" json:"-"` + Result LDAPIdentityResult `xml:"AssumeRoleWithLDAPIdentityResult"` ResponseMetadata struct { RequestID string `xml:"RequestId,omitempty"` } `xml:"ResponseMetadata,omitempty"` From 6d4f5e1eec3cfcdc988a6817c1d135bf522ac6cc Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Mon, 23 Sep 2019 23:52:17 +0000 Subject: [PATCH 063/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 2a51d3286..1daf055f2 100644 --- a/api.go +++ b/api.go @@ -104,7 +104,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.37" + libraryVersion = "v6.0.38" ) // User Agent should always following the below style. From 331a6cd23a8e3f79a72c0f6ab6af9d3562d53f25 Mon Sep 17 00:00:00 2001 From: Ashish Kumar Sinha Date: Sun, 29 Sep 2019 20:19:19 +0530 Subject: [PATCH 064/215] Core: Expose Response header in GetObject (#1166) Fixes #1106 --- .travis.yml | 3 ++- api-get-object-file.go | 2 +- api-get-object.go | 18 ++++++++--------- core.go | 5 +++-- core_test.go | 39 ++++++++++++++++++++++++++--------- functional_tests.go | 46 +++++++++++++++++++++--------------------- 6 files changed, 67 insertions(+), 46 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6400cc528..e50cefee7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,8 @@ matrix: before_install: - sudo apt-get install devscripts - - go get github.com/minio/minio + - mkdir /tmp/minio + - (cd /tmp/minio; GO111MODULE=on go get github.com/minio/minio) - sudo cp testcerts/public.crt /usr/local/share/ca-certificates/ - sudo update-ca-certificates - MINIO_ACCESS_KEY=minio MINIO_SECRET_KEY=minio123 ${GOPATH}/bin/minio server --compat --quiet --certs-dir testcerts data 2>&1 > minio.log & diff --git a/api-get-object-file.go b/api-get-object-file.go index 9c82a7c4f..0684b2b3a 100644 --- a/api-get-object-file.go +++ b/api-get-object-file.go @@ -100,7 +100,7 @@ func (c Client) fGetObjectWithContext(ctx context.Context, bucketName, objectNam } // Seek to current position for incoming reader. - objectReader, objectStat, err := c.getObject(ctx, bucketName, objectName, opts) + objectReader, objectStat, _, err := c.getObject(ctx, bucketName, objectName, opts) if err != nil { return err } diff --git a/api-get-object.go b/api-get-object.go index 5d03ea9bb..136782806 100644 --- a/api-get-object.go +++ b/api-get-object.go @@ -92,7 +92,7 @@ func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName } else if req.Offset > 0 { opts.SetRange(req.Offset, 0) } - httpReader, objectInfo, err = c.getObject(ctx, bucketName, objectName, opts) + httpReader, objectInfo, _, err = c.getObject(ctx, bucketName, objectName, opts) if err != nil { resCh <- getResponse{Error: err} return @@ -173,7 +173,7 @@ func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName } else if req.Offset > 0 { // Range is set with respect to the offset. opts.SetRange(req.Offset, 0) } - httpReader, objectInfo, err = c.getObject(ctx, bucketName, objectName, opts) + httpReader, objectInfo, _, err = c.getObject(ctx, bucketName, objectName, opts) if err != nil { resCh <- getResponse{ Error: err, @@ -596,13 +596,13 @@ func newObject(reqCh chan<- getRequest, resCh <-chan getResponse, doneCh chan<- // // For more information about the HTTP Range header. // go to http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35. -func (c Client) getObject(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, error) { +func (c Client) getObject(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, http.Header, error) { // Validate input arguments. if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return nil, ObjectInfo{}, err + return nil, ObjectInfo{}, nil, err } if err := s3utils.CheckValidObjectName(objectName); err != nil { - return nil, ObjectInfo{}, err + return nil, ObjectInfo{}, nil, err } // Execute GET on objectName. @@ -613,11 +613,11 @@ func (c Client) getObject(ctx context.Context, bucketName, objectName string, op contentSHA256Hex: emptySHA256Hex, }) if err != nil { - return nil, ObjectInfo{}, err + return nil, ObjectInfo{}, nil, err } if resp != nil { if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { - return nil, ObjectInfo{}, httpRespToErrorResponse(resp, bucketName, objectName) + return nil, ObjectInfo{}, nil, httpRespToErrorResponse(resp, bucketName, objectName) } } @@ -629,7 +629,7 @@ func (c Client) getObject(ctx context.Context, bucketName, objectName string, op date, err := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified")) if err != nil { msg := "Last-Modified time format not recognized. " + reportIssue - return nil, ObjectInfo{}, ErrorResponse{ + return nil, ObjectInfo{}, nil, ErrorResponse{ Code: "InternalError", Message: msg, RequestID: resp.Header.Get("x-amz-request-id"), @@ -657,5 +657,5 @@ func (c Client) getObject(ctx context.Context, bucketName, objectName string, op } // do not close body here, caller will close - return resp.Body, objectStat, nil + return resp.Body, objectStat, resp.Header, nil } diff --git a/core.go b/core.go index 4f5719343..9dae1a3ba 100644 --- a/core.go +++ b/core.go @@ -20,6 +20,7 @@ package minio import ( "context" "io" + "net/http" "strings" "github.com/minio/minio-go/v6/pkg/encrypt" @@ -176,14 +177,14 @@ func (c Core) PutBucketPolicy(bucket, bucketPolicy string) error { // GetObjectWithContext is a lower level API implemented to support reading // partial objects and also downloading objects with special conditions // matching etag, modtime etc. -func (c Core) GetObjectWithContext(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, error) { +func (c Core) GetObjectWithContext(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, http.Header, error) { return c.getObject(ctx, bucketName, objectName, opts) } // GetObject is a lower level API implemented to support reading // partial objects and also downloading objects with special conditions // matching etag, modtime etc. -func (c Core) GetObject(bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, error) { +func (c Core) GetObject(bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, http.Header, error) { return c.GetObjectWithContext(context.Background(), bucketName, objectName, opts) } diff --git a/core_test.go b/core_test.go index b171bf822..e22d5ae34 100644 --- a/core_test.go +++ b/core_test.go @@ -21,7 +21,9 @@ import ( "bytes" "io" "log" + "net/http" "os" + "strconv" "testing" "time" @@ -121,7 +123,7 @@ func TestGetObjectCore(t *testing.T) { opts := GetObjectOptions{} opts.SetRange(offset, offset+int64(len(buf1))-1) - reader, objectInfo, err := c.GetObject(bucketName, objectName, opts) + reader, objectInfo, _, err := c.GetObject(bucketName, objectName, opts) if err != nil { t.Fatal(err) } @@ -140,7 +142,7 @@ func TestGetObjectCore(t *testing.T) { offset += 512 opts.SetRange(offset, offset+int64(len(buf2))-1) - reader, objectInfo, err = c.GetObject(bucketName, objectName, opts) + reader, objectInfo, _, err = c.GetObject(bucketName, objectName, opts) if err != nil { t.Fatal(err) } @@ -159,7 +161,7 @@ func TestGetObjectCore(t *testing.T) { } opts.SetRange(0, int64(len(buf3))) - reader, objectInfo, err = c.GetObject(bucketName, objectName, opts) + reader, objectInfo, _, err = c.GetObject(bucketName, objectName, opts) if err != nil { t.Fatal(err) } @@ -180,7 +182,7 @@ func TestGetObjectCore(t *testing.T) { opts = GetObjectOptions{} opts.SetMatchETag("etag") - _, _, err = c.GetObject(bucketName, objectName, opts) + _, _, _, err = c.GetObject(bucketName, objectName, opts) if err == nil { t.Fatal("Unexpected GetObject should fail with mismatching etags") } @@ -190,7 +192,7 @@ func TestGetObjectCore(t *testing.T) { opts = GetObjectOptions{} opts.SetMatchETagExcept("etag") - reader, objectInfo, err = c.GetObject(bucketName, objectName, opts) + reader, objectInfo, _, err = c.GetObject(bucketName, objectName, opts) if err != nil { t.Fatal(err) } @@ -210,7 +212,7 @@ func TestGetObjectCore(t *testing.T) { opts = GetObjectOptions{} opts.SetRange(0, 0) - reader, objectInfo, err = c.GetObject(bucketName, objectName, opts) + reader, objectInfo, _, err = c.GetObject(bucketName, objectName, opts) if err != nil { t.Fatal(err) } @@ -225,6 +227,23 @@ func TestGetObjectCore(t *testing.T) { t.Fatalf("Error: GetObject read shorter bytes before reaching EOF, want %v, got %v\n", objectInfo.Size, m) } + opts = GetObjectOptions{} + opts.SetRange(offset, offset+int64(len(buf2))-1) + contentLength := len(buf2) + var header http.Header + _, _, header, err = c.GetObject(bucketName, objectName, opts) + if err != nil { + t.Fatal(err) + } + + contentLengthValue, err := strconv.Atoi(header.Get("Content-Length")) + if err != nil { + t.Fatal("Error: ", err) + } + if contentLength != contentLengthValue { + t.Fatalf("Error: Content Length in response header %v, not equal to set content lenght %v\n", contentLengthValue, contentLength) + } + err = c.RemoveObject(bucketName, objectName) if err != nil { t.Fatal("Error: ", err) @@ -287,7 +306,7 @@ func TestGetObjectContentEncoding(t *testing.T) { t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) } - rwc, objInfo, err := c.GetObject(bucketName, objectName, GetObjectOptions{}) + rwc, objInfo, _, err := c.GetObject(bucketName, objectName, GetObjectOptions{}) if err != nil { t.Fatalf("Error: %v", err) } @@ -590,7 +609,7 @@ func TestCoreCopyObjectPart(t *testing.T) { // Now we read the data back getOpts := GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) - r, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { t.Fatal("Error:", err, destBucketName, destObjectName) } @@ -604,7 +623,7 @@ func TestCoreCopyObjectPart(t *testing.T) { } getOpts.SetRange(5*1024*1024, 0) - r, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { t.Fatal("Error:", err, destBucketName, destObjectName) } @@ -765,7 +784,7 @@ func TestCoreGetObjectMetadata(t *testing.T) { log.Fatalln(err) } - reader, objInfo, err := core.GetObject(bucketName, "my-objectname", GetObjectOptions{}) + reader, objInfo, _, err := core.GetObject(bucketName, "my-objectname", GetObjectOptions{}) if err != nil { log.Fatalln(err) } diff --git a/functional_tests.go b/functional_tests.go index 30199cfe0..2df4b1da8 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -6245,7 +6245,7 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, } // 3. get copied object and check if content is equal coreClient := minio.Core{c} - reader, _, err := coreClient.GetObject(bucketName, "dstObject", minio.GetObjectOptions{ServerSideEncryption: dstEncryption}) + reader, _, _, err := coreClient.GetObject(bucketName, "dstObject", minio.GetObjectOptions{ServerSideEncryption: dstEncryption}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return @@ -6285,7 +6285,7 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, } // Get copied object and check if content is equal - reader, _, err = coreClient.GetObject(bucketName, "srcObject", minio.GetObjectOptions{ServerSideEncryption: newSSE}) + reader, _, _, err = coreClient.GetObject(bucketName, "srcObject", minio.GetObjectOptions{ServerSideEncryption: newSSE}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return @@ -6319,7 +6319,7 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, } // Get copied decrypted object and check if content is equal - reader, _, err = coreClient.GetObject(bucketName, "srcObject", minio.GetObjectOptions{}) + reader, _, _, err = coreClient.GetObject(bucketName, "srcObject", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return @@ -6814,7 +6814,7 @@ func testSSECMultipartEncryptedToSSECCopyObjectPart() { // Now we read the data back getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} getOpts.SetRange(0, 6*1024*1024-1) - r, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -6828,7 +6828,7 @@ func testSSECMultipartEncryptedToSSECCopyObjectPart() { } getOpts.SetRange(6*1024*1024, 0) - r, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -6965,7 +6965,7 @@ func testSSECEncryptedToSSECCopyObjectPart() { // Now we read the data back getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} getOpts.SetRange(0, 5*1024*1024-1) - r, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -6979,7 +6979,7 @@ func testSSECEncryptedToSSECCopyObjectPart() { } getOpts.SetRange(5*1024*1024, 0) - r, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -7115,7 +7115,7 @@ func testSSECEncryptedToUnencryptedCopyPart() { // Now we read the data back getOpts := minio.GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) - r, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -7129,7 +7129,7 @@ func testSSECEncryptedToUnencryptedCopyPart() { } getOpts.SetRange(5*1024*1024, 0) - r, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -7267,7 +7267,7 @@ func testSSECEncryptedToSSES3CopyObjectPart() { // Now we read the data back getOpts := minio.GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) - r, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -7281,7 +7281,7 @@ func testSSECEncryptedToSSES3CopyObjectPart() { } getOpts.SetRange(5*1024*1024, 0) - r, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -7416,7 +7416,7 @@ func testUnencryptedToSSECCopyObjectPart() { // Now we read the data back getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} getOpts.SetRange(0, 5*1024*1024-1) - r, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -7430,7 +7430,7 @@ func testUnencryptedToSSECCopyObjectPart() { } getOpts.SetRange(5*1024*1024, 0) - r, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -7562,7 +7562,7 @@ func testUnencryptedToUnencryptedCopyPart() { // Now we read the data back getOpts := minio.GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) - r, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -7576,7 +7576,7 @@ func testUnencryptedToUnencryptedCopyPart() { } getOpts.SetRange(5*1024*1024, 0) - r, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -7711,7 +7711,7 @@ func testUnencryptedToSSES3CopyObjectPart() { // Now we read the data back getOpts := minio.GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) - r, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -7725,7 +7725,7 @@ func testUnencryptedToSSES3CopyObjectPart() { } getOpts.SetRange(5*1024*1024, 0) - r, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -7860,7 +7860,7 @@ func testSSES3EncryptedToSSECCopyObjectPart() { // Now we read the data back getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} getOpts.SetRange(0, 5*1024*1024-1) - r, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -7874,7 +7874,7 @@ func testSSES3EncryptedToSSECCopyObjectPart() { } getOpts.SetRange(5*1024*1024, 0) - r, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -8007,7 +8007,7 @@ func testSSES3EncryptedToUnencryptedCopyPart() { // Now we read the data back getOpts := minio.GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) - r, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -8021,7 +8021,7 @@ func testSSES3EncryptedToUnencryptedCopyPart() { } getOpts.SetRange(5*1024*1024, 0) - r, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -8157,7 +8157,7 @@ func testSSES3EncryptedToSSES3CopyObjectPart() { // Now we read the data back getOpts := minio.GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) - r, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -8171,7 +8171,7 @@ func testSSES3EncryptedToSSES3CopyObjectPart() { } getOpts.SetRange(5*1024*1024, 0) - r, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } From 05a12f89cb8cfd65b7de5ea049548e2067496bb2 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Mon, 30 Sep 2019 17:36:28 -0700 Subject: [PATCH 065/215] Export Select Result Parser (#1170) This will make it possible to parse responses from custom requests. Very useful for tests. --- api-select.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api-select.go b/api-select.go index b5ce13112..7ecd95380 100644 --- a/api-select.go +++ b/api-select.go @@ -251,6 +251,12 @@ func (c Client) SelectObjectContent(ctx context.Context, bucketName, objectName return nil, err } + return NewSelectResults(resp, bucketName) +} + +// NewSelectResults creates a Select Result parser that parses the response +// and returns a Reader that will return parsed and assembled select output. +func NewSelectResults(resp *http.Response, bucketName string) (*SelectResults, error) { if resp.StatusCode != http.StatusOK { return nil, httpRespToErrorResponse(resp, bucketName, "") } From 23d7c54de4de9f50f5422b41f70e00d6704ae012 Mon Sep 17 00:00:00 2001 From: Anis Elleuch Date: Tue, 1 Oct 2019 02:17:45 +0100 Subject: [PATCH 066/215] list: Decode key/prefix and next marker in Core layer (#1169) Some modules depending on minio-go, such as Minio Gateway S3, depends on minio-go Core to perform listing. However a regression is found because decoding keys and common prefixes, along with marker for list v1 is not happening in Core module but in higher level. This PR moves decoding to Core layer. This PR also avoids encoding next continuation token since S3 spec does not require it. --- api-list.go | 148 ++++++++++++++++++++++++---------------------------- 1 file changed, 69 insertions(+), 79 deletions(-) diff --git a/api-list.go b/api-list.go index 0c6a4681f..2bd83feda 100644 --- a/api-list.go +++ b/api-list.go @@ -128,13 +128,6 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d // If contents are available loop through and send over channel. for _, object := range result.Contents { - object.Key, err = url.QueryUnescape(object.Key) - if err != nil { - objectStatCh <- ObjectInfo{ - Err: err, - } - return - } select { // Send object content. case objectStatCh <- object: @@ -147,17 +140,9 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d // Send all common prefixes if any. // NOTE: prefixes are only present if the request is delimited. for _, obj := range result.CommonPrefixes { - objInfo := ObjectInfo{} - objInfo.Key, err = url.QueryUnescape(obj.Prefix) - if err != nil { - objectStatCh <- ObjectInfo{ - Err: err, - } - return - } select { // Send object prefixes. - case objectStatCh <- objInfo: + case objectStatCh <- ObjectInfo{Key: obj.Prefix}: // If receives done from the caller, return here. case <-doneCh: return @@ -166,13 +151,7 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d // If continuation token present, save it for next request. if result.NextContinuationToken != "" { - continuationToken, err = url.QueryUnescape(result.NextContinuationToken) - if err != nil { - objectStatCh <- ObjectInfo{ - Err: err, - } - return - } + continuationToken = result.NextContinuationToken } // Listing ends result is not truncated, return right here. @@ -269,6 +248,20 @@ func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken s return listBucketResult, errors.New("Truncated response should have continuation token set") } + for _, obj := range listBucketResult.Contents { + obj.Key, err = url.QueryUnescape(obj.Key) + if err != nil { + return listBucketResult, err + } + } + + for _, obj := range listBucketResult.CommonPrefixes { + obj.Prefix, err = url.QueryUnescape(obj.Prefix) + if err != nil { + return listBucketResult, err + } + } + // Success. return listBucketResult, nil } @@ -338,13 +331,6 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don // If contents are available loop through and send over channel. for _, object := range result.Contents { - object.Key, err = url.QueryUnescape(object.Key) - if err != nil { - objectStatCh <- ObjectInfo{ - Err: err, - } - return - } // Save the marker. marker = object.Key select { @@ -359,18 +345,9 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don // Send all common prefixes if any. // NOTE: prefixes are only present if the request is delimited. for _, obj := range result.CommonPrefixes { - object := ObjectInfo{} - object.Key, err = url.QueryUnescape(obj.Prefix) - if err != nil { - objectStatCh <- ObjectInfo{ - Err: err, - } - return - } - object.Size = 0 select { // Send object prefixes. - case objectStatCh <- object: + case objectStatCh <- ObjectInfo{Key: obj.Prefix}: // If receives done from the caller, return here. case <-doneCh: return @@ -379,13 +356,7 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don // If next marker present, save it for next request. if result.NextMarker != "" { - marker, err = url.QueryUnescape(result.NextMarker) - if err != nil { - objectStatCh <- ObjectInfo{ - Err: err, - } - return - } + marker = result.NextMarker } // Listing ends result is not truncated, return right here. @@ -461,6 +432,28 @@ func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimit if err != nil { return listBucketResult, err } + + for _, obj := range listBucketResult.Contents { + obj.Key, err = url.QueryUnescape(obj.Key) + if err != nil { + return listBucketResult, err + } + } + + for _, obj := range listBucketResult.CommonPrefixes { + obj.Prefix, err = url.QueryUnescape(obj.Prefix) + if err != nil { + return listBucketResult, err + } + } + + if listBucketResult.NextMarker != "" { + listBucketResult.NextMarker, err = url.QueryUnescape(listBucketResult.NextMarker) + if err != nil { + return listBucketResult, err + } + } + return listBucketResult, nil } @@ -531,21 +524,9 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive } return } - // Save objectMarker and uploadIDMarker for next request. - objectMarker, err = url.QueryUnescape(result.NextKeyMarker) - if err != nil { - objectMultipartStatCh <- ObjectMultipartInfo{ - Err: err, - } - return - } - uploadIDMarker, err = url.QueryUnescape(result.NextUploadIDMarker) - if err != nil { - objectMultipartStatCh <- ObjectMultipartInfo{ - Err: err, - } - return - } + objectMarker = result.NextKeyMarker + uploadIDMarker = result.NextUploadIDMarker + // Send all multipart uploads. for _, obj := range result.Uploads { // Calculate total size of the uploaded parts if 'aggregateSize' is enabled. @@ -559,13 +540,6 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive continue } } - obj.Key, err = url.QueryUnescape(obj.Key) - if err != nil { - objectMultipartStatCh <- ObjectMultipartInfo{ - Err: err, - } - return - } select { // Send individual uploads here. case objectMultipartStatCh <- obj: @@ -577,18 +551,9 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive // Send all common prefixes if any. // NOTE: prefixes are only present if the request is delimited. for _, obj := range result.CommonPrefixes { - object := ObjectMultipartInfo{} - object.Key, err = url.QueryUnescape(obj.Prefix) - if err != nil { - objectMultipartStatCh <- ObjectMultipartInfo{ - Err: err, - } - return - } - object.Size = 0 select { // Send delimited prefixes here. - case objectMultipartStatCh <- object: + case objectMultipartStatCh <- ObjectMultipartInfo{Key: obj.Prefix, Size: 0}: // If done channel return here. case <-doneCh: return @@ -666,6 +631,31 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker, if err != nil { return listMultipartUploadsResult, err } + + listMultipartUploadsResult.NextKeyMarker, err = url.QueryUnescape(listMultipartUploadsResult.NextKeyMarker) + if err != nil { + return listMultipartUploadsResult, err + } + + listMultipartUploadsResult.NextUploadIDMarker, err = url.QueryUnescape(listMultipartUploadsResult.NextUploadIDMarker) + if err != nil { + return listMultipartUploadsResult, err + } + + for _, obj := range listMultipartUploadsResult.Uploads { + obj.Key, err = url.QueryUnescape(obj.Key) + if err != nil { + return listMultipartUploadsResult, err + } + } + + for _, obj := range listMultipartUploadsResult.CommonPrefixes { + obj.Prefix, err = url.QueryUnescape(obj.Prefix) + if err != nil { + return listMultipartUploadsResult, err + } + } + return listMultipartUploadsResult, nil } From 6248fe12cc011e6095aeee6ec59d733ea60d7c62 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Wed, 2 Oct 2019 00:58:19 +0000 Subject: [PATCH 067/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 1daf055f2..63a54e8e3 100644 --- a/api.go +++ b/api.go @@ -104,7 +104,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.38" + libraryVersion = "v6.0.39" ) // User Agent should always following the below style. From bb93a2c4f0cb3f2357e0dbfcf9d196bbf20c7fcb Mon Sep 17 00:00:00 2001 From: Anis Elleuch Date: Fri, 4 Oct 2019 18:36:09 +0100 Subject: [PATCH 068/215] list: Fix encoding in listing results (#1171) Add unit tests for objects name which contains special characters, such as, space, % and \x17.. --- api-list.go | 24 +++++------ functional_tests.go | 102 +++++++++++++++++++++----------------------- go.sum | 3 ++ 3 files changed, 64 insertions(+), 65 deletions(-) diff --git a/api-list.go b/api-list.go index 2bd83feda..14ec90722 100644 --- a/api-list.go +++ b/api-list.go @@ -248,15 +248,15 @@ func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken s return listBucketResult, errors.New("Truncated response should have continuation token set") } - for _, obj := range listBucketResult.Contents { - obj.Key, err = url.QueryUnescape(obj.Key) + for i, obj := range listBucketResult.Contents { + listBucketResult.Contents[i].Key, err = url.QueryUnescape(obj.Key) if err != nil { return listBucketResult, err } } - for _, obj := range listBucketResult.CommonPrefixes { - obj.Prefix, err = url.QueryUnescape(obj.Prefix) + for i, obj := range listBucketResult.CommonPrefixes { + listBucketResult.CommonPrefixes[i].Prefix, err = url.QueryUnescape(obj.Prefix) if err != nil { return listBucketResult, err } @@ -433,15 +433,15 @@ func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimit return listBucketResult, err } - for _, obj := range listBucketResult.Contents { - obj.Key, err = url.QueryUnescape(obj.Key) + for i, obj := range listBucketResult.Contents { + listBucketResult.Contents[i].Key, err = url.QueryUnescape(obj.Key) if err != nil { return listBucketResult, err } } - for _, obj := range listBucketResult.CommonPrefixes { - obj.Prefix, err = url.QueryUnescape(obj.Prefix) + for i, obj := range listBucketResult.CommonPrefixes { + listBucketResult.CommonPrefixes[i].Prefix, err = url.QueryUnescape(obj.Prefix) if err != nil { return listBucketResult, err } @@ -642,15 +642,15 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker, return listMultipartUploadsResult, err } - for _, obj := range listMultipartUploadsResult.Uploads { - obj.Key, err = url.QueryUnescape(obj.Key) + for i, obj := range listMultipartUploadsResult.Uploads { + listMultipartUploadsResult.Uploads[i].Key, err = url.QueryUnescape(obj.Key) if err != nil { return listMultipartUploadsResult, err } } - for _, obj := range listMultipartUploadsResult.CommonPrefixes { - obj.Prefix, err = url.QueryUnescape(obj.Prefix) + for i, obj := range listMultipartUploadsResult.CommonPrefixes { + listMultipartUploadsResult.CommonPrefixes[i].Prefix, err = url.QueryUnescape(obj.Prefix) if err != nil { return listMultipartUploadsResult, err } diff --git a/functional_tests.go b/functional_tests.go index 2df4b1da8..69ae3b2b0 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -9969,7 +9969,7 @@ func testFGetObjectWithContextV2() { } -// Test list object v1 and V2 storage class fields +// Test list object v1 and V2 func testListObjects() { // initialize logging params startTime := time.Now() @@ -10012,67 +10012,64 @@ func testListObjects() { return } - bufSize := dataFileMap["datafile-33-kB"] - var reader = getDataReader("datafile-33-kB") - defer reader.Close() - - // Save the data - objectName1 := randString(60, rand.NewSource(time.Now().UnixNano()), "") + testObjects := []struct { + name string + storageClass string + }{ - _, err = c.PutObject(bucketName, objectName1, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", StorageClass: "STANDARD"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject1 call failed", err) - return + // \x17 is a forbidden character in a xml document + {"foo\x17bar", "STANDARD"}, + // Special characters + {"foo bar", "STANDARD"}, + {"foo-%", "STANDARD"}, + {"random-object-1", "STANDARD"}, + {"random-object-2", "REDUCED_REDUNDANCY"}, + } + + for i, object := range testObjects { + bufSize := dataFileMap["datafile-33-kB"] + var reader = getDataReader("datafile-33-kB") + defer reader.Close() + _, err = c.PutObject(bucketName, object.name, reader, int64(bufSize), + minio.PutObjectOptions{ContentType: "binary/octet-stream", StorageClass: object.storageClass}) + if err != nil { + logError(testName, function, args, startTime, "", fmt.Sprintf("PutObject %d call failed", i+1), err) + return + } } - bufSize1 := dataFileMap["datafile-33-kB"] - var reader1 = getDataReader("datafile-33-kB") - defer reader1.Close() - objectName2 := randString(60, rand.NewSource(time.Now().UnixNano()), "") + testList := func(listFn func(string, string, bool, <-chan struct{}) <-chan minio.ObjectInfo, bucket string) { + // Create a done channel to control 'ListObjects' go routine. + doneCh := make(chan struct{}) + // Exit cleanly upon return. + defer close(doneCh) - _, err = c.PutObject(bucketName, objectName2, reader1, int64(bufSize1), minio.PutObjectOptions{ContentType: "binary/octet-stream", StorageClass: "REDUCED_REDUNDANCY"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject2 call failed", err) - return - } + var objCursor int - // Create a done channel to control 'ListObjects' go routine. - doneCh := make(chan struct{}) - // Exit cleanly upon return. - defer close(doneCh) - - // check for storage-class from ListObjects result - for objInfo := range c.ListObjects(bucketName, "", true, doneCh) { - if objInfo.Err != nil { - logError(testName, function, args, startTime, "", "ListObjects failed unexpectedly", err) - return - } - if objInfo.Key == objectName1 && objInfo.StorageClass != "STANDARD" { - // Ignored as Gateways (Azure/GCS etc) wont return storage class - ignoredLog(testName, function, args, startTime, "ListObjects doesn't return expected storage class").Info() - } - if objInfo.Key == objectName2 && objInfo.StorageClass != "REDUCED_REDUNDANCY" { - // Ignored as Gateways (Azure/GCS etc) wont return storage class - ignoredLog(testName, function, args, startTime, "ListObjects doesn't return expected storage class").Info() + // check for object name and storage-class from listing object result + for objInfo := range listFn(bucket, "", true, doneCh) { + if objInfo.Err != nil { + logError(testName, function, args, startTime, "", "ListObjects failed unexpectedly", err) + return + } + if objInfo.Key != testObjects[objCursor].name { + logError(testName, function, args, startTime, "", "ListObjects does not return expected object name", err) + } + if objInfo.StorageClass != testObjects[objCursor].storageClass { + // Ignored as Gateways (Azure/GCS etc) wont return storage class + ignoredLog(testName, function, args, startTime, "ListObjects doesn't return expected storage class").Info() + } + objCursor++ } - } - // check for storage-class from ListObjectsV2 result - for objInfo := range c.ListObjectsV2(bucketName, "", true, doneCh) { - if objInfo.Err != nil { - logError(testName, function, args, startTime, "", "ListObjectsV2 failed unexpectedly", err) - return - } - if objInfo.Key == objectName1 && objInfo.StorageClass != "STANDARD" { - // Ignored as Gateways (Azure/GCS etc) wont return storage class - ignoredLog(testName, function, args, startTime, "ListObjectsV2 doesn't return expected storage class").Info() - } - if objInfo.Key == objectName2 && objInfo.StorageClass != "REDUCED_REDUNDANCY" { - // Ignored as Gateways (Azure/GCS etc) wont return storage class - ignoredLog(testName, function, args, startTime, "ListObjectsV2 doesn't return expected storage class").Info() + if objCursor != len(testObjects) { + logError(testName, function, args, startTime, "", "ListObjects returned unexpected number of items", errors.New("")) } } + testList(c.ListObjects, bucketName) + testList(c.ListObjectsV2, bucketName) + // Delete all objects and buckets if err = cleanupBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "Cleanup failed", err) @@ -10080,7 +10077,6 @@ func testListObjects() { } successLogger(testName, function, args, startTime).Info() - } // Convert string to bool and always return false if any error diff --git a/go.sum b/go.sum index b3cacbda4..cd02277ed 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -10,6 +11,7 @@ github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKU github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -18,6 +20,7 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo= From e11840609c348b5d1e46185b7f6e98ce6e6bc313 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Tue, 8 Oct 2019 17:51:21 +0000 Subject: [PATCH 069/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 63a54e8e3..71cbc1196 100644 --- a/api.go +++ b/api.go @@ -104,7 +104,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.39" + libraryVersion = "v6.0.40" ) // User Agent should always following the below style. From fba2959cbee546eb5397e23c613656cfdf232488 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 24 Oct 2019 17:04:35 -0700 Subject: [PATCH 070/215] Send error for json.Unmarshal for caller action (#1179) This change is needed such that applications can detect errors and act appropriately while using ListenBucketNotifications when server unexpectedly disconnects. --- api-notification.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/api-notification.go b/api-notification.go index f35619541..0480c21eb 100644 --- a/api-notification.go +++ b/api-notification.go @@ -200,6 +200,11 @@ func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, even for bio.Scan() { var notificationInfo NotificationInfo if err = json.Unmarshal(bio.Bytes(), ¬ificationInfo); err != nil { + // Unexpected error during json unmarshal, send + // the error to caller for actionable as needed. + notificationInfoCh <- NotificationInfo{ + Err: err, + } closeResponse(resp) continue } @@ -211,7 +216,11 @@ func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, even return } } - + if err = bio.Err(); err != nil { + notificationInfoCh <- NotificationInfo{ + Err: err, + } + } // Close current connection before looping further. closeResponse(resp) } From 4fc01063647b4ce455eab487e20ab63504784a63 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 25 Oct 2019 21:44:41 -0700 Subject: [PATCH 071/215] Set max-keys query param only as needed (#1180) This is to ensure that S3 server implementations can send a larger list response by default i.e more than 1000 keys if needed. --- api-list.go | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/api-list.go b/api-list.go index 14ec90722..b9b0bcccd 100644 --- a/api-list.go +++ b/api-list.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2015-2019 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -208,12 +208,10 @@ func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken s urlValues.Set("fetch-owner", "true") } - // maxkeys should default to 1000 or less. - if maxkeys == 0 || maxkeys > 1000 { - maxkeys = 1000 - } // Set max keys. - urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys)) + if maxkeys > 0 { + urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys)) + } // Set start-after if startAfter != "" { @@ -401,12 +399,10 @@ func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimit urlValues.Set("marker", objectMarker) } - // maxkeys should default to 1000 or less. - if maxkeys == 0 || maxkeys > 1000 { - maxkeys = 1000 - } // Set max keys. - urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys)) + if maxkeys > 0 { + urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys)) + } // Always set encoding-type urlValues.Set("encoding-type", "url") From 731a57a65a085ef848b4ae0da3fc8d55735d4aa6 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Wed, 30 Oct 2019 23:23:16 +0000 Subject: [PATCH 072/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 71cbc1196..559fc62a2 100644 --- a/api.go +++ b/api.go @@ -104,7 +104,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.40" + libraryVersion = "v6.0.41" ) // User Agent should always following the below style. From b80ac7085a93aaf4a4a3ade1a02585d0bf8393a6 Mon Sep 17 00:00:00 2001 From: Bala FA Date: Fri, 1 Nov 2019 22:27:39 +0000 Subject: [PATCH 073/215] Add user metadata headers into ObjectInfo (#1151) Now StatObject calls returns ObjectInfo contains user metadata which is stripped "x-amz-meta-" prefix with first value. Fixes #1145 --- api-datatypes.go | 3 +++ api-stat.go | 14 +++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/api-datatypes.go b/api-datatypes.go index 3fb47fc10..9974ca46f 100644 --- a/api-datatypes.go +++ b/api-datatypes.go @@ -47,6 +47,9 @@ type ObjectInfo struct { // eg: x-amz-meta-*, content-encoding etc. Metadata http.Header `json:"metadata" xml:"-"` + // x-amz-meta-* headers stripped "x-amz-meta-" prefix containing the first value. + UserMetadata map[string]string `json:"userMetadata" xml:"-"` + // Owner name. Owner struct { DisplayName string `json:"name"` diff --git a/api-stat.go b/api-stat.go index c6fb47d63..5cc40e757 100644 --- a/api-stat.go +++ b/api-stat.go @@ -176,6 +176,17 @@ func (c Client) statObject(ctx context.Context, bucketName, objectName string, o if t, err := time.Parse(http.TimeFormat, expiryStr); err == nil { expTime = t.UTC() } + + metadata := extractObjMetadata(resp.Header) + userMetadata := map[string]string{} + const xamzmeta = "x-amz-meta-" + const xamzmetaLen = len(xamzmeta) + for k, v := range metadata { + if strings.HasPrefix(strings.ToLower(k), xamzmeta) { + userMetadata[k[xamzmetaLen:]] = v[0] + } + } + // Save object metadata info. return ObjectInfo{ ETag: md5sum, @@ -187,6 +198,7 @@ func (c Client) statObject(ctx context.Context, bucketName, objectName string, o // Extract only the relevant header keys describing the object. // following function filters out a list of standard set of keys // which are not part of object metadata. - Metadata: extractObjMetadata(resp.Header), + Metadata: metadata, + UserMetadata: userMetadata, }, nil } From 30047d6a7c2e5df9b7c464e9d0c6dfbd5b63bf13 Mon Sep 17 00:00:00 2001 From: Bala FA Date: Sun, 10 Nov 2019 09:26:01 +0530 Subject: [PATCH 074/215] add object lock configuration APIs (#1153) --- api-object-lock.go | 228 +++++++++++++++++++++++ api-put-bucket.go | 93 ++++++++- docs/API.md | 182 ++++++++++++++++-- examples/s3/disableversioning.go | 47 +++++ examples/s3/enableversioning.go | 47 +++++ examples/s3/getbucketobjectlockconfig.go | 56 ++++++ examples/s3/makebucketwithobjectlock.go | 47 +++++ examples/s3/setbucketobjectlockconfig.go | 54 ++++++ 8 files changed, 735 insertions(+), 19 deletions(-) create mode 100644 api-object-lock.go create mode 100644 examples/s3/disableversioning.go create mode 100644 examples/s3/enableversioning.go create mode 100644 examples/s3/getbucketobjectlockconfig.go create mode 100644 examples/s3/makebucketwithobjectlock.go create mode 100644 examples/s3/setbucketobjectlockconfig.go diff --git a/api-object-lock.go b/api-object-lock.go new file mode 100644 index 000000000..15f13a543 --- /dev/null +++ b/api-object-lock.go @@ -0,0 +1,228 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "context" + "encoding/xml" + "fmt" + "net/http" + "net/url" + "time" + + "github.com/minio/minio-go/v6/pkg/s3utils" +) + +// RetentionMode - object retention mode. +type RetentionMode string + +const ( + // Governance - goverance mode. + Governance RetentionMode = "GOVERNANCE" + + // Compliance - compliance mode. + Compliance RetentionMode = "COMPLIANCE" +) + +func (r RetentionMode) String() string { + return string(r) +} + +// IsValid - check whether this retention mode is valid or not. +func (r RetentionMode) IsValid() bool { + return r == Governance || r == Compliance +} + +// ValidityUnit - retention validity unit. +type ValidityUnit string + +const ( + // Days - denotes no. of days. + Days ValidityUnit = "DAYS" + + // Years - denotes no. of years. + Years ValidityUnit = "YEARS" +) + +func (unit ValidityUnit) String() string { + return string(unit) +} + +// IsValid - check whether this validity unit is valid or not. +func (unit ValidityUnit) isValid() bool { + return unit == Days || unit == Years +} + +// Retention - bucket level retention configuration. +type Retention struct { + Mode RetentionMode + Validity time.Duration +} + +func (r Retention) String() string { + return fmt.Sprintf("{Mode:%v, Validity:%v}", r.Mode, r.Validity) +} + +// IsEmpty - returns whether retention is empty or not. +func (r Retention) IsEmpty() bool { + return r.Mode == "" || r.Validity == 0 +} + +// objectLockConfig - object lock configuration specified in +// https://docs.aws.amazon.com/AmazonS3/latest/API/Type_API_ObjectLockConfiguration.html +type objectLockConfig struct { + XMLNS string `xml:"xmlns,attr,omitempty"` + XMLName xml.Name `xml:"ObjectLockConfiguration"` + ObjectLockEnabled string `xml:"ObjectLockEnabled"` + Rule *struct { + DefaultRetention struct { + Mode RetentionMode `xml:"Mode"` + Days *uint `xml:"Days"` + Years *uint `xml:"Years"` + } `xml:"DefaultRetention"` + } `xml:"Rule,omitempty"` +} + +func newObjectLockConfig(mode *RetentionMode, validity *uint, unit *ValidityUnit) (*objectLockConfig, error) { + config := &objectLockConfig{ + ObjectLockEnabled: "Enabled", + } + + if mode != nil && validity != nil && unit != nil { + if !mode.IsValid() { + return nil, fmt.Errorf("invalid retention mode `%v`", mode) + } + + if !unit.isValid() { + return nil, fmt.Errorf("invalid validity unit `%v`", unit) + } + + config.Rule = &struct { + DefaultRetention struct { + Mode RetentionMode `xml:"Mode"` + Days *uint `xml:"Days"` + Years *uint `xml:"Years"` + } `xml:"DefaultRetention"` + }{} + + config.Rule.DefaultRetention.Mode = *mode + if *unit == Days { + config.Rule.DefaultRetention.Days = validity + } else { + config.Rule.DefaultRetention.Years = validity + } + + return config, nil + } + + if mode == nil && validity == nil && unit == nil { + return config, nil + } + + return nil, fmt.Errorf("all of retention mode, validity and validity unit must be passed") +} + +// SetBucketObjectLockConfig sets object lock configuration in given bucket. mode, validity and unit are either all set or all nil. +func (c Client) SetBucketObjectLockConfig(bucketName string, mode *RetentionMode, validity *uint, unit *ValidityUnit) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("object-lock", "") + + config, err := newObjectLockConfig(mode, validity, unit) + if err != nil { + return err + } + + configData, err := xml.Marshal(config) + if err != nil { + return err + } + + reqMetadata := requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: bytes.NewReader(configData), + contentLength: int64(len(configData)), + contentMD5Base64: sumMD5Base64(configData), + contentSHA256Hex: sum256Hex(configData), + } + + // Execute PUT bucket object lock configuration. + resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, bucketName, "") + } + } + return nil +} + +// GetBucketObjectLockConfig gets object lock configuration of given bucket. +func (c Client) GetBucketObjectLockConfig(bucketName string) (mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return nil, nil, nil, err + } + + urlValues := make(url.Values) + urlValues.Set("object-lock", "") + + // Execute GET on bucket to list objects. + resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return nil, nil, nil, err + } + + config := &objectLockConfig{} + if err = xml.NewDecoder(resp.Body).Decode(config); err != nil { + return nil, nil, nil, err + } + + if config.Rule != nil { + mode = &config.Rule.DefaultRetention.Mode + if config.Rule.DefaultRetention.Days != nil { + validity = config.Rule.DefaultRetention.Days + days := Days + unit = &days + } else { + validity = config.Rule.DefaultRetention.Years + years := Years + unit = &years + } + + return mode, validity, unit, nil + } + + return nil, nil, nil, nil +} diff --git a/api-put-bucket.go b/api-put-bucket.go index 28613a229..0041ce179 100644 --- a/api-put-bucket.go +++ b/api-put-bucket.go @@ -31,14 +31,7 @@ import ( /// Bucket operations -// MakeBucket creates a new bucket with bucketName. -// -// Location is an optional argument, by default all buckets are -// created in US Standard Region. -// -// For Amazon S3 for more supported regions - http://docs.aws.amazon.com/general/latest/gr/rande.html -// For Google Cloud Storage for more supported regions - https://cloud.google.com/storage/docs/bucket-locations -func (c Client) MakeBucket(bucketName string, location string) (err error) { +func (c Client) makeBucket(bucketName string, location string, objectLockEnabled bool) (err error) { defer func() { // Save the location into cache on a successful makeBucket response. if err == nil { @@ -66,6 +59,12 @@ func (c Client) MakeBucket(bucketName string, location string) (err error) { bucketLocation: location, } + if objectLockEnabled { + headers := make(http.Header) + headers.Add("x-amz-bucket-object-lock-enabled", "true") + reqMetadata.customHeader = headers + } + // If location is not 'us-east-1' create bucket location config. if location != "us-east-1" && location != "" { createBucketConfig := createBucketConfiguration{} @@ -98,6 +97,28 @@ func (c Client) MakeBucket(bucketName string, location string) (err error) { return nil } +// MakeBucket creates a new bucket with bucketName. +// +// Location is an optional argument, by default all buckets are +// created in US Standard Region. +// +// For Amazon S3 for more supported regions - http://docs.aws.amazon.com/general/latest/gr/rande.html +// For Google Cloud Storage for more supported regions - https://cloud.google.com/storage/docs/bucket-locations +func (c Client) MakeBucket(bucketName string, location string) (err error) { + return c.makeBucket(bucketName, location, false) +} + +// MakeBucketWithObjectLock creates a object lock enabled new bucket with bucketName. +// +// Location is an optional argument, by default all buckets are +// created in US Standard Region. +// +// For Amazon S3 for more supported regions - http://docs.aws.amazon.com/general/latest/gr/rande.html +// For Google Cloud Storage for more supported regions - https://cloud.google.com/storage/docs/bucket-locations +func (c Client) MakeBucketWithObjectLock(bucketName string, location string) (err error) { + return c.makeBucket(bucketName, location, true) +} + // SetBucketPolicy set the access permissions on an existing bucket. func (c Client) SetBucketPolicy(bucketName, policy string) error { // Input validation. @@ -304,3 +325,59 @@ func (c Client) SetBucketNotification(bucketName string, bucketNotification Buck func (c Client) RemoveAllBucketNotification(bucketName string) error { return c.SetBucketNotification(bucketName, BucketNotification{}) } + +var ( + versionEnableConfig = []byte("Enabled") + versionEnableConfigLen = int64(len(versionEnableConfig)) + versionEnableConfigMD5Sum = sumMD5Base64(versionEnableConfig) + versionEnableConfigSHA256 = sum256Hex(versionEnableConfig) + + versionDisableConfig = []byte("Suspended") + versionDisableConfigLen = int64(len(versionDisableConfig)) + versionDisableConfigMD5Sum = sumMD5Base64(versionDisableConfig) + versionDisableConfigSHA256 = sum256Hex(versionDisableConfig) +) + +func (c Client) setVersioning(bucketName string, config []byte, length int64, md5sum, sha256sum string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("versioning", "") + + reqMetadata := requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: bytes.NewReader(config), + contentLength: length, + contentMD5Base64: md5sum, + contentSHA256Hex: sha256sum, + } + + // Execute PUT to set a bucket versioning. + resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, bucketName, "") + } + } + return nil +} + +// EnableVersioning - Enable object versioning in given bucket. +func (c Client) EnableVersioning(bucketName string) error { + return c.setVersioning(bucketName, versionEnableConfig, versionEnableConfigLen, versionEnableConfigMD5Sum, versionEnableConfigSHA256) +} + +// DisableVersioning - Disable object versioning in given bucket. +func (c Client) DisableVersioning(bucketName string) error { + return c.setVersioning(bucketName, versionDisableConfig, versionDisableConfigLen, versionDisableConfigMD5Sum, versionDisableConfigSHA256) +} diff --git a/docs/API.md b/docs/API.md index 76e1a6215..beb684d63 100644 --- a/docs/API.md +++ b/docs/API.md @@ -53,17 +53,17 @@ func main() { | Bucket operations | Object operations | Encrypted Object operations | Presigned operations | Bucket Policy/Notification Operations | Client custom settings | | :--- | :--- | :--- | :--- | :--- | :--- | | [`MakeBucket`](#MakeBucket) | [`GetObject`](#GetObject) | [`GetObject`](#GetObject) | [`PresignedGetObject`](#PresignedGetObject) | [`SetBucketPolicy`](#SetBucketPolicy) | [`SetAppInfo`](#SetAppInfo) | -| [`ListBuckets`](#ListBuckets) | [`PutObject`](#PutObject) | [`PutObject`](#PutObject) | [`PresignedPutObject`](#PresignedPutObject) | [`GetBucketPolicy`](#GetBucketPolicy) | [`SetCustomTransport`](#SetCustomTransport) | -| [`BucketExists`](#BucketExists) | [`CopyObject`](#CopyObject) | [`CopyObject`](#CopyObject) | [`PresignedPostPolicy`](#PresignedPostPolicy) | [`SetBucketNotification`](#SetBucketNotification) | [`TraceOn`](#TraceOn) | -| [`RemoveBucket`](#RemoveBucket) | [`StatObject`](#StatObject) | [`StatObject`](#StatObject) | | [`GetBucketNotification`](#GetBucketNotification) | [`TraceOff`](#TraceOff) | -| [`ListObjects`](#ListObjects) | [`RemoveObject`](#RemoveObject) | | | [`RemoveAllBucketNotification`](#RemoveAllBucketNotification) | [`SetS3TransferAccelerate`](#SetS3TransferAccelerate) | -| [`ListObjectsV2`](#ListObjectsV2) | [`RemoveObjects`](#RemoveObjects) | | | [`ListenBucketNotification`](#ListenBucketNotification) | | -| [`ListIncompleteUploads`](#ListIncompleteUploads) | [`RemoveIncompleteUpload`](#RemoveIncompleteUpload) | | | [`SetBucketLifecycle`](#SetBucketLifecycle) | | -| | [`FPutObject`](#FPutObject) | [`FPutObject`](#FPutObject) | | [`GetBucketLifecycle`](#GetBucketLifecycle) | | -| | [`FGetObject`](#FGetObject) | [`FGetObject`](#FGetObject) | | | | -| | [`ComposeObject`](#ComposeObject) | [`ComposeObject`](#ComposeObject) | | | | -| | [`NewSourceInfo`](#NewSourceInfo) | [`NewSourceInfo`](#NewSourceInfo) | | | | -| | [`NewDestinationInfo`](#NewDestinationInfo) | [`NewDestinationInfo`](#NewDestinationInfo) | | | | +| [`MakeBucketWithObjectLock`](#MakeBucketWithObjectLock) | [`PutObject`](#PutObject) | [`PutObject`](#PutObject) | [`PresignedPutObject`](#PresignedPutObject) | [`GetBucketPolicy`](#GetBucketPolicy) | [`SetCustomTransport`](#SetCustomTransport) | +| [`ListBuckets`](#ListBuckets) | [`CopyObject`](#CopyObject) | [`CopyObject`](#CopyObject) | [`PresignedPostPolicy`](#PresignedPostPolicy) | [`SetBucketNotification`](#SetBucketNotification) | [`TraceOn`](#TraceOn) | +| [`BucketExists`](#BucketExists) | [`StatObject`](#StatObject) | [`StatObject`](#StatObject) | | [`GetBucketNotification`](#GetBucketNotification) | [`TraceOff`](#TraceOff) | +| [`RemoveBucket`](#RemoveBucket) | [`RemoveObject`](#RemoveObject) | | | [`RemoveAllBucketNotification`](#RemoveAllBucketNotification) | [`SetS3TransferAccelerate`](#SetS3TransferAccelerate) | +| [`ListObjects`](#ListObjects) | [`RemoveObjects`](#RemoveObjects) | | | [`ListenBucketNotification`](#ListenBucketNotification) | | +| [`ListObjectsV2`](#ListObjectsV2) | [`RemoveIncompleteUpload`](#RemoveIncompleteUpload) | | | [`SetBucketLifecycle`](#SetBucketLifecycle) | | +| [`ListIncompleteUploads`](#ListIncompleteUploads) | [`FPutObject`](#FPutObject) | [`FPutObject`](#FPutObject) | | [`GetBucketLifecycle`](#GetBucketLifecycle) | | +| | [`FGetObject`](#FGetObject) | [`FGetObject`](#FGetObject) | | [`SetBucketObjectLockConfig`](#SetBucketObjectLockConfig) | | +| | [`ComposeObject`](#ComposeObject) | [`ComposeObject`](#ComposeObject) | | [`GetBucketObjectLockConfig`](#GetBucketObjectLockConfig) | | +| | [`NewSourceInfo`](#NewSourceInfo) | [`NewSourceInfo`](#NewSourceInfo) | | [`EnableVersioning`](#EnableVersioning) | | +| | [`NewDestinationInfo`](#NewDestinationInfo) | [`NewDestinationInfo`](#NewDestinationInfo) | | [`DisableVersioning`](#DisableVersioning) | | | | [`PutObjectWithContext`](#PutObjectWithContext) | [`PutObjectWithContext`](#PutObjectWithContext) | | | | | [`GetObjectWithContext`](#GetObjectWithContext) | [`GetObjectWithContext`](#GetObjectWithContext) | | | | | [`FPutObjectWithContext`](#FPutObjectWithContext) | [`FPutObjectWithContext`](#FPutObjectWithContext) | | | @@ -144,6 +144,39 @@ if err != nil { fmt.Println("Successfully created mybucket.") ``` + +### MakeBucketWithObjectLock(bucketName, location string) error +Creates a new bucket with object lock enabled. + +__Parameters__ + +| Param | Type | Description | +|---|---|---| +|`bucketName` | _string_ | Name of the bucket | +| `location` | _string_ | Region where the bucket is to be created. Default value is us-east-1. Other valid values are listed below. Note: When used with minio server, use the region specified in its config file (defaults to us-east-1).| +| | |us-east-1 | +| | |us-west-1 | +| | |us-west-2 | +| | |eu-west-1 | +| | | eu-central-1| +| | | ap-southeast-1| +| | | ap-northeast-1| +| | | ap-southeast-2| +| | | sa-east-1| + + +__Example__ + + +```go +err = minioClient.MakeBucketWithObjectLock("mybucket", "us-east-1") +if err != nil { + fmt.Println(err) + return +} +fmt.Println("Successfully created mybucket.") +``` + ### ListBuckets() ([]BucketInfo, error) Lists all buckets. @@ -1565,6 +1598,133 @@ if err != nil { } ``` + +### SetBucketObjectLockConfig(bucketname, mode *RetentionMode, validity *uint, unit *ValidityUnit) error +Set object lock configuration in given bucket. mode, validity and unit are either all set or all nil. + +__Parameters__ + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket| +|`mode` | _RetentionMode_ |Retention mode to be set | +|`validity` | _uint_ |Validity period to be set | +|`unit` | _ValidityUnit_ |Unit of validity period | + +__Return Values__ + +|Param |Type |Description | +|:---|:---| :---| +|`err` | _error_ |Standard Error | + +__Example__ + +```go +mode := Governance +validity := uint(30) +unit := Days + +err = minioClient.SetBucketObjectLockConfig("my-bucketname", &mode, &validity, &unit) +if err != nil { + fmt.Println(err) + return +} +``` + + +### GetBucketObjectLockConfig(bucketName) (*RetentionMode, *uint, *ValidityUnit, error) +Get object lock configuration of given bucket. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | + +__Return Values__ + + +|Param |Type |Description | +|:---|:---| :---| +|`mode` | _RetentionMode_ |Current retention mode | +|`validity` | _uint_ |Current validity period | +|`unit` | _ValidityUnit_ |Unit of validity period | +|`err` | _error_ |Standard Error | + +__Example__ + +```go +mode, validity, unit, err := minioClient.GetObjectLockConfig("my-bucketname") +if err != nil { + log.Fatalln(err) +} + +if mode != nil { + fmt.Printf("%v mode is enabled for %v %v for bucket 'my-bucketname'\n", *mode, *validity, *unit) +} else { + fmt.Println("No mode is enabled for bucket 'my-bucketname'") +} +``` + + +### EnableVersioning(bucketName) error +Enable bucket versioning support. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | + +__Return Values__ + + +|Param |Type |Description | +|:---|:---| :---| +|`err` | _error_ |Standard Error | + +__Example__ + +```go +err := minioClient.EnableVersioning("my-bucketname") +if err != nil { + log.Fatalln(err) +} + +fmt.Println("versioning enabled for bucket 'my-bucketname'") +``` + + +### DisableVersioning(bucketName) error +Disable bucket versioning support. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | + +__Return Values__ + + +|Param |Type |Description | +|:---|:---| :---| +|`err` | _error_ |Standard Error | + +__Example__ + +```go +err := minioClient.DisableVersioning("my-bucketname") +if err != nil { + log.Fatalln(err) +} + +fmt.Println("versioning disabled for bucket 'my-bucketname'") +``` + ## 7. Client custom settings diff --git a/examples/s3/disableversioning.go b/examples/s3/disableversioning.go new file mode 100644 index 000000000..bbaec5e0c --- /dev/null +++ b/examples/s3/disableversioning.go @@ -0,0 +1,47 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + minio "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + err = s3Client.DisableVersioning("my-bucketname") + if err != nil { + log.Fatalln(err) + } + log.Println("Disabled") +} diff --git a/examples/s3/enableversioning.go b/examples/s3/enableversioning.go new file mode 100644 index 000000000..58179be36 --- /dev/null +++ b/examples/s3/enableversioning.go @@ -0,0 +1,47 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + minio "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + err = s3Client.EnableVersioning("my-bucketname") + if err != nil { + log.Fatalln(err) + } + log.Println("Enabled") +} diff --git a/examples/s3/getbucketobjectlockconfig.go b/examples/s3/getbucketobjectlockconfig.go new file mode 100644 index 000000000..ddd9f9250 --- /dev/null +++ b/examples/s3/getbucketobjectlockconfig.go @@ -0,0 +1,56 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + "log" + + minio "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // s3Client.TraceOn(os.Stderr) + + // Get object lock configuration. + mode, validity, unit, err := s3Client.GetBucketObjectLockConfig("my-bucketname") + if err != nil { + log.Fatalln(err) + } + + if mode != nil { + fmt.Printf("%v mode is enabled for %v %v for bucket 'my-bucketname'\n", *mode, *validity, *unit) + } else { + fmt.Println("No mode is enabled for bucket 'my-bucketname'") + } +} diff --git a/examples/s3/makebucketwithobjectlock.go b/examples/s3/makebucketwithobjectlock.go new file mode 100644 index 000000000..af52dd02b --- /dev/null +++ b/examples/s3/makebucketwithobjectlock.go @@ -0,0 +1,47 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + minio "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + err = s3Client.MakeBucketWithObjectLock("my-bucketname", "us-east-1") + if err != nil { + log.Fatalln(err) + } + log.Println("Success") +} diff --git a/examples/s3/setbucketobjectlockconfig.go b/examples/s3/setbucketobjectlockconfig.go new file mode 100644 index 000000000..8034969a5 --- /dev/null +++ b/examples/s3/setbucketobjectlockconfig.go @@ -0,0 +1,54 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + minio "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // s3Client.TraceOn(os.Stderr) + + // Set object lock configuration. + mode := minio.Governance + validity := uint(30) + unit := minio.Days + + err = s3Client.SetBucketObjectLockConfig("my-bucketname", &mode, &validity, &unit) + if err != nil { + log.Fatalln(err) + } + log.Println("Success") +} From 437215bf4b6f14ae8344ed60ade08296b0d0a753 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Mon, 11 Nov 2019 02:50:23 +0000 Subject: [PATCH 075/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 559fc62a2..69bd17059 100644 --- a/api.go +++ b/api.go @@ -104,7 +104,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.41" + libraryVersion = "v6.0.42" ) // User Agent should always following the below style. From d930d6bb77f3691c0f75cadb2b8768b13a3f47d5 Mon Sep 17 00:00:00 2001 From: kannappanr <30541348+kannappanr@users.noreply.github.com> Date: Wed, 20 Nov 2019 10:11:05 -0800 Subject: [PATCH 076/215] Add object retention support (#1187) --- api-object-lock.go | 6 +- api-object-retention.go | 168 +++++++++++++++++++++++++++++ api-put-object.go | 16 +++ api-remove.go | 29 +++++ docs/API.md | 93 ++++++++++++++++ examples/s3/getobjectretention.go | 48 +++++++++ examples/s3/putobjectretention.go | 54 ++++++++++ examples/s3/removeobjectoptions.go | 49 +++++++++ utils.go | 2 + 9 files changed, 464 insertions(+), 1 deletion(-) create mode 100644 api-object-retention.go create mode 100644 examples/s3/getobjectretention.go create mode 100644 examples/s3/putobjectretention.go create mode 100644 examples/s3/removeobjectoptions.go diff --git a/api-object-lock.go b/api-object-lock.go index 15f13a543..c30ab3258 100644 --- a/api-object-lock.go +++ b/api-object-lock.go @@ -203,7 +203,11 @@ func (c Client) GetBucketObjectLockConfig(bucketName string) (mode *RetentionMod if err != nil { return nil, nil, nil, err } - + if resp != nil { + if resp.StatusCode != http.StatusOK { + return nil, nil, nil, httpRespToErrorResponse(resp, bucketName, "") + } + } config := &objectLockConfig{} if err = xml.NewDecoder(resp.Body).Decode(config); err != nil { return nil, nil, nil, err diff --git a/api-object-retention.go b/api-object-retention.go new file mode 100644 index 000000000..8aa08f073 --- /dev/null +++ b/api-object-retention.go @@ -0,0 +1,168 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "context" + "encoding/xml" + "fmt" + "net/http" + "net/url" + "time" + + "github.com/minio/minio-go/v6/pkg/s3utils" +) + +// objectRetention - object retention specified in +// https://docs.aws.amazon.com/AmazonS3/latest/API/Type_API_ObjectLockConfiguration.html +type objectRetention struct { + XMLNS string `xml:"xmlns,attr,omitempty"` + XMLName xml.Name `xml:"Retention"` + Mode RetentionMode `xml:"Mode"` + RetainUntilDate time.Time `type:"timestamp" timestampFormat:"iso8601" xml:"RetainUntilDate"` +} + +func newObjectRetention(mode *RetentionMode, date *time.Time) (*objectRetention, error) { + if mode == nil { + return nil, fmt.Errorf("Mode not set") + } + + if date == nil { + return nil, fmt.Errorf("RetainUntilDate not set") + } + + if !mode.IsValid() { + return nil, fmt.Errorf("invalid retention mode `%v`", mode) + } + objectRetention := &objectRetention{ + Mode: *mode, + RetainUntilDate: *date, + } + return objectRetention, nil +} + +// PutObjectRetentionOptions represents options specified by user for PutObject call +type PutObjectRetentionOptions struct { + GovernanceBypass bool + Mode *RetentionMode + RetainUntilDate *time.Time + VersionID string +} + +// PutObjectRetention : sets object retention for a given object and versionID. +func (c Client) PutObjectRetention(bucketName, objectName string, opts PutObjectRetentionOptions) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("retention", "") + + if opts.VersionID != "" { + urlValues.Set("versionId", opts.VersionID) + } + + retention, err := newObjectRetention(opts.Mode, opts.RetainUntilDate) + if err != nil { + return err + } + + retentionData, err := xml.Marshal(retention) + if err != nil { + return err + } + + // Build headers. + headers := make(http.Header) + + if opts.GovernanceBypass { + // Set the bypass goverenance retention header + headers.Set("x-amz-bypass-governance-retention", "True") + } + + reqMetadata := requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: urlValues, + contentBody: bytes.NewReader(retentionData), + contentLength: int64(len(retentionData)), + contentMD5Base64: sumMD5Base64(retentionData), + contentSHA256Hex: sum256Hex(retentionData), + customHeader: headers, + } + + // Execute PUT Object Retention. + resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp != nil { + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return httpRespToErrorResponse(resp, bucketName, objectName) + } + } + return nil +} + +// GetObjectRetention gets retention of given object. +func (c Client) GetObjectRetention(bucketName, objectName, versionID string) (mode *RetentionMode, retainUntilDate *time.Time, err error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return nil, nil, err + } + + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return nil, nil, err + } + urlValues := make(url.Values) + urlValues.Set("retention", "") + if versionID != "" { + urlValues.Set("versionId", versionID) + } + // Execute GET on bucket to list objects. + resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return nil, nil, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return nil, nil, httpRespToErrorResponse(resp, bucketName, objectName) + } + } + retention := &objectRetention{} + if err = xml.NewDecoder(resp.Body).Decode(retention); err != nil { + return nil, nil, err + } + + return &retention.Mode, &retention.RetainUntilDate, nil +} diff --git a/api-put-object.go b/api-put-object.go index 8f33fca6c..bc0848bbf 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -25,6 +25,7 @@ import ( "net/http" "runtime/debug" "sort" + "time" "github.com/minio/minio-go/v6/pkg/encrypt" "github.com/minio/minio-go/v6/pkg/s3utils" @@ -40,6 +41,8 @@ type PutObjectOptions struct { ContentDisposition string ContentLanguage string CacheControl string + Mode *RetentionMode + RetainUntilDate *time.Time ServerSideEncryption encrypt.ServerSide NumThreads uint StorageClass string @@ -80,6 +83,14 @@ func (opts PutObjectOptions) Header() (header http.Header) { if opts.CacheControl != "" { header["Cache-Control"] = []string{opts.CacheControl} } + + if opts.Mode != nil { + header["x-amz-object-lock-mode"] = []string{opts.Mode.String()} + } + if opts.RetainUntilDate != nil { + header["x-amz-object-lock-retain-until-date"] = []string{opts.RetainUntilDate.Format(time.RFC3339)} + } + if opts.ServerSideEncryption != nil { opts.ServerSideEncryption.Marshal(header) } @@ -109,6 +120,11 @@ func (opts PutObjectOptions) validate() (err error) { return ErrInvalidArgument(v + " unsupported user defined metadata value") } } + if opts.Mode != nil { + if !opts.Mode.IsValid() { + return ErrInvalidArgument(opts.Mode.String() + " unsupported retention mode") + } + } return nil } diff --git a/api-remove.go b/api-remove.go index e919be17d..4c8c335c3 100644 --- a/api-remove.go +++ b/api-remove.go @@ -60,6 +60,17 @@ func (c Client) RemoveBucket(bucketName string) error { // RemoveObject remove an object from a bucket. func (c Client) RemoveObject(bucketName, objectName string) error { + return c.RemoveObjectWithOptions(bucketName, objectName, RemoveObjectOptions{}) +} + +// RemoveObjectOptions represents options specified by user for PutObject call +type RemoveObjectOptions struct { + GovernanceBypass bool + VersionID string +} + +// RemoveObjectWithOptions removes an object from a bucket. +func (c Client) RemoveObjectWithOptions(bucketName, objectName string, opts RemoveObjectOptions) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -67,11 +78,29 @@ func (c Client) RemoveObject(bucketName, objectName string) error { if err := s3utils.CheckValidObjectName(objectName); err != nil { return err } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + + if opts.VersionID != "" { + urlValues.Set("versionId", opts.VersionID) + } + + // Build headers. + headers := make(http.Header) + + if opts.GovernanceBypass { + // Set the bypass goverenance retention header + headers.Set("x-amz-bypass-governance-retention", "True") + } // Execute DELETE on objectName. resp, err := c.executeMethod(context.Background(), "DELETE", requestMetadata{ bucketName: bucketName, objectName: objectName, contentSHA256Hex: emptySHA256Hex, + queryValues: urlValues, + customHeader: headers, }) defer closeResponse(resp) if err != nil { diff --git a/docs/API.md b/docs/API.md index beb684d63..f17487de9 100644 --- a/docs/API.md +++ b/docs/API.md @@ -69,6 +69,9 @@ func main() { | | [`FPutObjectWithContext`](#FPutObjectWithContext) | [`FPutObjectWithContext`](#FPutObjectWithContext) | | | | | [`FGetObjectWithContext`](#FGetObjectWithContext) | [`FGetObjectWithContext`](#FGetObjectWithContext) | | | | | [`RemoveObjectsWithContext`](#RemoveObjectsWithContext) | | | | +| | [`RemoveObjectWithOptions`](#RemoveObjectWithOptions) | | | | +| | [`PutObjectRetention`](#PutObjectRetention) | | | | +| | [`GetObjectRetention`](#GetObjectRetention) | | | | | | [`SelectObjectContent`](#SelectObjectContent) | | ## 1. Constructor @@ -585,6 +588,8 @@ __minio.PutObjectOptions__ | `opts.ContentDisposition` | _string_ | Content disposition of object, "inline" | | `opts.ContentLanguage` | _string_ | Content language of object, e.g "French" | | `opts.CacheControl` | _string_ | Used to specify directives for caching mechanisms in both requests and responses e.g "max-age=600"| +| `opts.Mode` | _*minio.RetentionMode_ | Retention mode to be set, e.g "COMPLIANCE" | +| `opts.RetainUntilDate` | _*time.Time_ | Time until which the retention applied is valid| | `opts.ServerSideEncryption` | _encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v6) | | `opts.StorageClass` | _string_ | Specify storage class for the object. Supported values for MinIO server are `REDUCED_REDUNDANCY` and `STANDARD` | | `opts.WebsiteRedirectLocation` | _string_ | Specify a redirect for the object, to another object in the same bucket or to a external URL. | @@ -1102,6 +1107,94 @@ for rErr := range minioClient.RemoveObjects(ctx, "my-bucketname", objectsCh) { fmt.Println("Error detected during deletion: ", rErr) } ``` + +### RemoveObjectWithOptions(bucketName, objectName string, opts minio.RemoveObjectOptions) error +Removes an object. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`opts` |_minio.RemoveObjectOptions_ |Allows user to set options | + +__minio.RemoveObjectOptions__ + +|Field | Type | Description | +|:--- |:--- | :--- | +| `opts.GovernanceBypass` | _bool_ |Set the bypass governance header to delete an object locked with GOVERNANCE mode| +| `opts.VersionID` | _string_ |Version ID of the object to delete| + + +```go +opts := minio.RemoveObjectOptions { + GovernanceBypass: true, + VersionID: "myversionid", + } +err = minioClient.RemoveObjectWithOptions("mybucket", "myobject", opts) +if err != nil { + fmt.Println(err) + return +} +``` + +### PutObjectRetention(bucketName, objectName string, opts minio.PutObjectRetentionOptions) error +Applies object retention lock onto an object. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`opts` |_minio.PutObjectRetentionOptions_ |Allows user to set options like retention mode, expiry date and version id | + +__minio.PutObjectRetentionOptions__ + +|Field | Type | Description | +|:--- |:--- | :--- | +| `opts.GovernanceBypass` | _bool_ |Set the bypass governance header to overwrite object retention if the existing retention mode is set to GOVERNANCE| +| `opts.Mode` | _*minio.RetentionMode_ |Retention mode to be set| +| `opts.RetainUntilDate` | _*time.Time_ |Time until which the retention applied is valid| +| `opts.VersionID` | _string_ |Version ID of the object to apply retention on| + +```go +t := time.Date(2020, time.November, 18, 14, 0, 0, 0, time.UTC) +m := minio.RetentionMode(minio.Compliance) +opts := minio.PutObjectRetentionOptions { + GovernanceBypass: true, + RetainUntilDate: &t, + Mode: &m, + } +err = minioClient.PutObjectRetention("mybucket", "myobject", opts) +if err != nil { + fmt.Println(err) + return +} +``` + +### GetObjectRetention(bucketName, objectName, versionID string) (mode *RetentionMode, retainUntilDate *time.Time, err error) +Returns retention set on a given object. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`versionID` |_string_ |Version ID of the object | + +```go +err = minioClient.PutObjectRetention("mybucket", "myobject", "") +if err != nil { + fmt.Println(err) + return +} +``` ### SelectObjectContent(ctx context.Context, bucketName string, objectsName string, expression string, options SelectObjectOptions) *SelectResults Parameters diff --git a/examples/s3/getobjectretention.go b/examples/s3/getobjectretention.go new file mode 100644 index 000000000..527f8a2ec --- /dev/null +++ b/examples/s3/getobjectretention.go @@ -0,0 +1,48 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "io" + "log" + "os" + + "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and + // my-testfile are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + if err != nil { + log.Fatalln(err) + } + m, t, err = s3Client.GetObjectRetention("my-bucket", "my-object", "") + if err != nil { + log.Fatalln(err) + } + log.Println("Get object retention successful, Mode: ", m.String(), " Retainuntil Date ", t.String()) +} diff --git a/examples/s3/putobjectretention.go b/examples/s3/putobjectretention.go new file mode 100644 index 000000000..b0db21a5b --- /dev/null +++ b/examples/s3/putobjectretention.go @@ -0,0 +1,54 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + "time" + + "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and + // my-testfile are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + if err != nil { + log.Fatalln(err) + } + t := time.Date(2020, time.November, 18, 14, 0, 0, 0, time.UTC) + m := minio.RetentionMode(minio.Governance) + opts := minio.PutObjectRetentionOptions{ + GovernanceBypass: true, + RetainUntilDate: &t, + Mode: &m, + } + err = s3Client.PutObjectRetention("my-bucket", "my-object", opts) + if err != nil { + log.Fatalln(err) + } + log.Println("Set object retention on my-object successfully.") +} diff --git a/examples/s3/removeobjectoptions.go b/examples/s3/removeobjectoptions.go new file mode 100644 index 000000000..926d80eed --- /dev/null +++ b/examples/s3/removeobjectoptions.go @@ -0,0 +1,49 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and + // my-testfile are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + if err != nil { + log.Fatalln(err) + } + opts := minio.RemoveObjectOptions{ + GovernanceBypass: true, + } + err = s3Client.RemoveObjectWithOptions("my-bucket", "my-object", opts) + if err != nil { + log.Fatalln(err) + } + log.Println("Remove object successful") +} diff --git a/utils.go b/utils.go index 2a743a834..d24cfb5c7 100644 --- a/utils.go +++ b/utils.go @@ -224,6 +224,8 @@ var supportedHeaders = []string{ "content-disposition", "content-language", "x-amz-website-redirect-location", + "x-amz-object-lock-mode", + "x-amz-object-lock-retain-until-date", "expires", // Add more supported headers here. } From 43607f8e1017dd1834c4e89c800c17c0e9c1ab12 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Wed, 20 Nov 2019 18:43:27 +0000 Subject: [PATCH 077/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 69bd17059..7d64a3d97 100644 --- a/api.go +++ b/api.go @@ -104,7 +104,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.42" + libraryVersion = "v6.0.43" ) // User Agent should always following the below style. From 9d428b0e0479eb1867bfb706480a30bc4b362601 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 21 Nov 2019 05:23:10 -0800 Subject: [PATCH 078/215] Support ListObjectsWithMetadata API (#1189) --- Makefile | 5 ++ api-datatypes.go | 34 ++++++++++- api-list.go | 73 ++++++++++++++++++------ core.go | 2 +- examples/s3/listobjectsV2WithMetadata.go | 58 +++++++++++++++++++ 5 files changed, 153 insertions(+), 19 deletions(-) create mode 100644 examples/s3/listobjectsV2WithMetadata.go diff --git a/Makefile b/Makefile index 5cfe33a46..18a74665c 100644 --- a/Makefile +++ b/Makefile @@ -15,3 +15,8 @@ examples: functional-test: @GO111MODULE=on SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minio SECRET_KEY=minio123 ENABLE_HTTPS=1 MINT_MODE=full go run functional_tests.go + +clean: + @echo "Cleaning up all the generated files" + @find . -name '*.test' | xargs rm -fv + @find . -name '*~' | xargs rm -fv diff --git a/api-datatypes.go b/api-datatypes.go index 9974ca46f..4e54f3415 100644 --- a/api-datatypes.go +++ b/api-datatypes.go @@ -18,6 +18,8 @@ package minio import ( + "encoding/xml" + "io" "net/http" "time" ) @@ -30,6 +32,36 @@ type BucketInfo struct { CreationDate time.Time `json:"creationDate"` } +// StringMap represents map with custom UnmarshalXML +type StringMap map[string]string + +// UnmarshalXML unmarshals the XML into a map of string to strings, +// creating a key in the map for each tag and setting it's value to the +// tags contents. +// +// The fact this function is on the pointer of Map is important, so that +// if m is nil it can be initialized, which is often the case if m is +// nested in another xml structurel. This is also why the first thing done +// on the first line is initialize it. +func (m *StringMap) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + *m = StringMap{} + type xmlMapEntry struct { + XMLName xml.Name + Value string `xml:",chardata"` + } + for { + var e xmlMapEntry + err := d.Decode(&e) + if err == io.EOF { + break + } else if err != nil { + return err + } + (*m)[e.XMLName.Local] = e.Value + } + return nil +} + // ObjectInfo container for object metadata. type ObjectInfo struct { // An ETag is optionally set to md5sum of an object. In case of multipart objects, @@ -48,7 +80,7 @@ type ObjectInfo struct { Metadata http.Header `json:"metadata" xml:"-"` // x-amz-meta-* headers stripped "x-amz-meta-" prefix containing the first value. - UserMetadata map[string]string `json:"userMetadata" xml:"-"` + UserMetadata StringMap `json:"userMetadata"` // Owner name. Owner struct { diff --git a/api-list.go b/api-list.go index b9b0bcccd..09c1565cb 100644 --- a/api-list.go +++ b/api-list.go @@ -60,9 +60,13 @@ func (c Client) ListBuckets() ([]BucketInfo, error) { /// Bucket Read Operations. -// ListObjectsV2 lists all objects matching the objectPrefix from -// the specified bucket. If recursion is enabled it would list -// all subdirectories and all its contents. +// ListObjectsV2WithMetadata lists all objects matching the objectPrefix +// from the specified bucket. If recursion is enabled it would list +// all subdirectories and all its contents. This call adds +// UserMetadata information as well for each object. +// +// This is a MinIO extension, this will not work against other S3 +// compatible object storage vendors. // // Your input parameters are just bucketName, objectPrefix, recursive // and a done channel for pro-actively closing the internal go @@ -76,11 +80,18 @@ func (c Client) ListBuckets() ([]BucketInfo, error) { // defer close(doneCh) // // Recursively list all objects in 'mytestbucket' // recursive := true -// for message := range api.ListObjectsV2("mytestbucket", "starthere", recursive, doneCh) { +// // Add metadata +// metadata := true +// for message := range api.ListObjectsV2WithMetadata("mytestbucket", "starthere", recursive, doneCh) { // fmt.Println(message) // } // -func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { +func (c Client) ListObjectsV2WithMetadata(bucketName, objectPrefix string, recursive bool, + doneCh <-chan struct{}) <-chan ObjectInfo { + return c.listObjectsV2(bucketName, objectPrefix, recursive, true, doneCh) +} + +func (c Client) listObjectsV2(bucketName, objectPrefix string, recursive, metadata bool, doneCh <-chan struct{}) <-chan ObjectInfo { // Allocate new list objects channel. objectStatCh := make(chan ObjectInfo, 1) // Default listing is delimited at "/" @@ -118,7 +129,8 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d var continuationToken string for { // Get list of objects a maximum of 1000 per request. - result, err := c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, 1000, "") + result, err := c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, + fetchOwner, metadata, delimiter, 0, "") if err != nil { objectStatCh <- ObjectInfo{ Err: err, @@ -163,6 +175,30 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d return objectStatCh } +// ListObjectsV2 lists all objects matching the objectPrefix from +// the specified bucket. If recursion is enabled it would list +// all subdirectories and all its contents. +// +// Your input parameters are just bucketName, objectPrefix, recursive +// and a done channel for pro-actively closing the internal go +// routine. If you enable recursive as 'true' this function will +// return back all the objects in a given bucket name and object +// prefix. +// +// api := client.New(....) +// // Create a done channel. +// doneCh := make(chan struct{}) +// defer close(doneCh) +// // Recursively list all objects in 'mytestbucket' +// recursive := true +// for message := range api.ListObjectsV2("mytestbucket", "starthere", recursive, doneCh) { +// fmt.Println(message) +// } +// +func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { + return c.listObjectsV2(bucketName, objectPrefix, recursive, false, doneCh) +} + // listObjectsV2Query - (List Objects V2) - List some or all (up to 1000) of the objects in a bucket. // // You can use the request parameters as selection criteria to return a subset of the objects in a bucket. @@ -173,7 +209,8 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d // ?prefix - Limits the response to keys that begin with the specified prefix. // ?max-keys - Sets the maximum number of keys returned in the response body. // ?start-after - Specifies the key to start after when listing objects in a bucket. -func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int, startAfter string) (ListBucketV2Result, error) { +// ?metadata - Specifies if we want metadata for the objects as part of list operation. +func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken string, fetchOwner, metadata bool, delimiter string, maxkeys int, startAfter string) (ListBucketV2Result, error) { // Validate bucket name. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ListBucketV2Result{}, err @@ -189,6 +226,10 @@ func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken s // Always set list-type in ListObjects V2 urlValues.Set("list-type", "2") + if metadata { + urlValues.Set("metadata", "true") + } + // Always set encoding-type in ListObjects V2 urlValues.Set("encoding-type", "url") @@ -319,7 +360,7 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don var marker string for { // Get list of objects a maximum of 1000 per request. - result, err := c.listObjectsQuery(bucketName, objectPrefix, marker, delimiter, 1000) + result, err := c.listObjectsQuery(bucketName, objectPrefix, marker, delimiter, 0) if err != nil { objectStatCh <- ObjectInfo{ Err: err, @@ -513,7 +554,7 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive var uploadIDMarker string for { // list all multipart uploads. - result, err := c.listMultipartUploadsQuery(bucketName, objectMarker, uploadIDMarker, objectPrefix, delimiter, 1000) + result, err := c.listMultipartUploadsQuery(bucketName, objectMarker, uploadIDMarker, objectPrefix, delimiter, 0) if err != nil { objectMultipartStatCh <- ObjectMultipartInfo{ Err: err, @@ -600,11 +641,10 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker, urlValues.Set("encoding-type", "url") // maxUploads should be 1000 or less. - if maxUploads == 0 || maxUploads > 1000 { - maxUploads = 1000 + if maxUploads > 0 { + // Set max-uploads. + urlValues.Set("max-uploads", fmt.Sprintf("%d", maxUploads)) } - // Set max-uploads. - urlValues.Set("max-uploads", fmt.Sprintf("%d", maxUploads)) // Execute GET on bucketName to list multipart uploads. resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ @@ -740,11 +780,10 @@ func (c Client) listObjectPartsQuery(bucketName, objectName, uploadID string, pa urlValues.Set("uploadId", uploadID) // maxParts should be 1000 or less. - if maxParts == 0 || maxParts > 1000 { - maxParts = 1000 + if maxParts > 0 { + // Set max parts. + urlValues.Set("max-parts", fmt.Sprintf("%d", maxParts)) } - // Set max parts. - urlValues.Set("max-parts", fmt.Sprintf("%d", maxParts)) // Execute GET on objectName to get list of parts. resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ diff --git a/core.go b/core.go index 9dae1a3ba..0d43ae0f5 100644 --- a/core.go +++ b/core.go @@ -53,7 +53,7 @@ func (c Core) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) // ListObjectsV2 - Lists all the objects at a prefix, similar to ListObjects() but uses // continuationToken instead of marker to support iteration over the results. func (c Core) ListObjectsV2(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int, startAfter string) (ListBucketV2Result, error) { - return c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, maxkeys, startAfter) + return c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, false, delimiter, maxkeys, startAfter) } // CopyObjectWithContext - copies an object from source object to destination object on server side. diff --git a/examples/s3/listobjectsV2WithMetadata.go b/examples/s3/listobjectsV2WithMetadata.go new file mode 100644 index 000000000..77eb50919 --- /dev/null +++ b/examples/s3/listobjectsV2WithMetadata.go @@ -0,0 +1,58 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + + "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-prefixname + // are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + fmt.Println(err) + return + } + + // Create a done channel to control 'ListObjects' go routine. + doneCh := make(chan struct{}) + + // Indicate to our routine to exit cleanly upon return. + defer close(doneCh) + + // List all objects from a bucket-name with a matching prefix. + for object := range s3Client.ListObjectsV2WithMetadata("my-bucketname", "my-prefixname", true, doneCh) { + if object.Err != nil { + fmt.Println(object.Err) + return + } + fmt.Println(object) + } + return +} From 3721161a9e0f2ae10a2d8f0da88938ab2eeb99c1 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Thu, 21 Nov 2019 17:12:59 +0000 Subject: [PATCH 079/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 7d64a3d97..f0978b633 100644 --- a/api.go +++ b/api.go @@ -104,7 +104,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.43" + libraryVersion = "v6.0.44" ) // User Agent should always following the below style. From d18cd1c3fde63effb2eb789120abdc73da4a5610 Mon Sep 17 00:00:00 2001 From: Rob Best Date: Mon, 25 Nov 2019 06:37:34 +0000 Subject: [PATCH 080/215] credentials: support AWS_CONTAINER_CREDENTIALS_FULL_URI (#1185) This environment variable allows the definition of a full URI to an ECS task endpoint on a loop-back address. It is documented sparsely compared to AWS_CONTAINER_CREDENTIALS_RELATIVE_URI but is implemented in every major AWS SDK. See: - https://github.com/aws/aws-sdk-go/blob/master/aws/defaults/defaults.go#L117 - https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/ECSCredentials.html - https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/EC2ContainerCredentialsProviderWrapper.html --- pkg/credentials/iam_aws.go | 54 +++++++++++++++++++++++++++++---- pkg/credentials/iam_aws_test.go | 30 ++++++++++++++++++ 2 files changed, 78 insertions(+), 6 deletions(-) diff --git a/pkg/credentials/iam_aws.go b/pkg/credentials/iam_aws.go index 5732f2e4b..34c593fdf 100644 --- a/pkg/credentials/iam_aws.go +++ b/pkg/credentials/iam_aws.go @@ -22,6 +22,7 @@ import ( "encoding/json" "errors" "fmt" + "net" "net/http" "net/url" "os" @@ -57,14 +58,35 @@ const ( ) // https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html -func getEndpoint(endpoint string) (string, bool) { +func getEndpoint(endpoint string) (string, bool, error) { + ecsFullURI := os.Getenv("AWS_CONTAINER_CREDENTIALS_FULL_URI") + ecsURI := os.Getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") + if endpoint != "" { - return endpoint, os.Getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") != "" + return endpoint, ecsURI != "" || ecsFullURI != "", nil + } + if ecsFullURI != "" { + u, err := url.Parse(ecsFullURI) + if err != nil { + return "", false, err + } + host := u.Hostname() + if host == "" { + return "", false, fmt.Errorf("can't parse host from uri: %s", ecsFullURI) + } + + if loopback, err := isLoopback(host); loopback { + return ecsFullURI, true, nil + } else if err != nil { + return "", false, err + } else { + return "", false, fmt.Errorf("host is not on a loopback address: %s", host) + } } - if ecsURI := os.Getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"); ecsURI != "" { - return fmt.Sprintf("%s%s", defaultECSRoleEndpoint, ecsURI), true + if ecsURI != "" { + return fmt.Sprintf("%s%s", defaultECSRoleEndpoint, ecsURI), true, nil } - return defaultIAMRoleEndpoint, false + return defaultIAMRoleEndpoint, false, nil } // NewIAM returns a pointer to a new Credentials object wrapping the IAM. @@ -82,9 +104,14 @@ func NewIAM(endpoint string) *Credentials { // Error will be returned if the request fails, or unable to extract // the desired func (m *IAM) Retrieve() (Value, error) { - endpoint, isEcsTask := getEndpoint(m.endpoint) var roleCreds ec2RoleCredRespBody var err error + + endpoint, isEcsTask, err := getEndpoint(m.endpoint) + if err != nil { + return Value{}, err + } + if isEcsTask { roleCreds, err = getEcsTaskCredentials(m.Client, endpoint) } else { @@ -248,3 +275,18 @@ func getCredentials(client *http.Client, endpoint string) (ec2RoleCredRespBody, return respCreds, nil } + +// isLoopback identifies if a host is on a loopback address +func isLoopback(host string) (bool, error) { + ips, err := net.LookupHost(host) + if err != nil { + return false, err + } + for _, ip := range ips { + if !net.ParseIP(ip).IsLoopback() { + return false, nil + } + } + + return true, nil +} diff --git a/pkg/credentials/iam_aws_test.go b/pkg/credentials/iam_aws_test.go index 90f980693..d502a8ace 100644 --- a/pkg/credentials/iam_aws_test.go +++ b/pkg/credentials/iam_aws_test.go @@ -243,3 +243,33 @@ func TestEcsTask(t *testing.T) { t.Error("Expected creds to be expired.") } } + +func TestEcsTaskFullURI(t *testing.T) { + server := initEcsTaskTestServer("2014-12-16T01:51:37Z") + defer server.Close() + p := &IAM{ + Client: http.DefaultClient, + } + os.Setenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", + fmt.Sprintf("%s%s", server.URL, "/v2/credentials?id=task_credential_id")) + creds, err := p.Retrieve() + os.Unsetenv("AWS_CONTAINER_CREDENTIALS_FULL_URI") + if err != nil { + t.Errorf("Unexpected failure %s", err) + } + if "accessKey" != creds.AccessKeyID { + t.Errorf("Expected \"accessKey\", got %s", creds.AccessKeyID) + } + + if "secret" != creds.SecretAccessKey { + t.Errorf("Expected \"secret\", got %s", creds.SecretAccessKey) + } + + if "token" != creds.SessionToken { + t.Errorf("Expected \"token\", got %s", creds.SessionToken) + } + + if !p.IsExpired() { + t.Error("Expected creds to be expired.") + } +} From 7bb4ac4dfa8be91227211bca4c0386e591d828b5 Mon Sep 17 00:00:00 2001 From: Pierre-Emmanuel Jacquier <15922119+pierre-emmanuelJ@users.noreply.github.com> Date: Tue, 26 Nov 2019 02:17:56 +0100 Subject: [PATCH 081/215] Add owner information in GetObjectACL and add Context (#1182) --- api-get-object-acl-context.go | 138 ++++++++++++++++++++++++++++++++++ api-get-object-acl.go | 111 +-------------------------- examples/s3/getobjectacl.go | 6 ++ functional_tests.go | 45 ++++++----- 4 files changed, 172 insertions(+), 128 deletions(-) create mode 100644 api-get-object-acl-context.go diff --git a/api-get-object-acl-context.go b/api-get-object-acl-context.go new file mode 100644 index 000000000..24d87671c --- /dev/null +++ b/api-get-object-acl-context.go @@ -0,0 +1,138 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2018 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "net/http" + "net/url" +) + +type accessControlPolicy struct { + Owner struct { + ID string `xml:"ID"` + DisplayName string `xml:"DisplayName"` + } `xml:"Owner"` + AccessControlList struct { + Grant []struct { + Grantee struct { + ID string `xml:"ID"` + DisplayName string `xml:"DisplayName"` + URI string `xml:"URI"` + } `xml:"Grantee"` + Permission string `xml:"Permission"` + } `xml:"Grant"` + } `xml:"AccessControlList"` +} + +//GetObjectACLWithContext get object ACLs +func (c Client) GetObjectACLWithContext(ctx context.Context, bucketName, objectName string) (*ObjectInfo, error) { + resp, err := c.executeMethod(ctx, "GET", requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: url.Values{ + "acl": []string{""}, + }, + }) + if err != nil { + return nil, err + } + defer closeResponse(resp) + + if resp.StatusCode != http.StatusOK { + return nil, httpRespToErrorResponse(resp, bucketName, objectName) + } + + res := &accessControlPolicy{} + + if err := xmlDecoder(resp.Body, res); err != nil { + return nil, err + } + + objInfo, err := c.statObject(ctx, bucketName, objectName, StatObjectOptions{}) + if err != nil { + return nil, err + } + + objInfo.Owner.DisplayName = res.Owner.DisplayName + objInfo.Owner.ID = res.Owner.ID + + cannedACL := getCannedACL(res) + if cannedACL != "" { + objInfo.Metadata.Add("X-Amz-Acl", cannedACL) + return &objInfo, nil + } + + grantACL := getAmzGrantACL(res) + for k, v := range grantACL { + objInfo.Metadata[k] = v + } + + return &objInfo, nil +} + +func getCannedACL(aCPolicy *accessControlPolicy) string { + grants := aCPolicy.AccessControlList.Grant + + switch { + case len(grants) == 1: + if grants[0].Grantee.URI == "" && grants[0].Permission == "FULL_CONTROL" { + return "private" + } + case len(grants) == 2: + for _, g := range grants { + if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" && g.Permission == "READ" { + return "authenticated-read" + } + if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && g.Permission == "READ" { + return "public-read" + } + if g.Permission == "READ" && g.Grantee.ID == aCPolicy.Owner.ID { + return "bucket-owner-read" + } + } + case len(grants) == 3: + for _, g := range grants { + if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && g.Permission == "WRITE" { + return "public-read-write" + } + } + } + return "" +} + +func getAmzGrantACL(aCPolicy *accessControlPolicy) map[string][]string { + grants := aCPolicy.AccessControlList.Grant + res := map[string][]string{} + + for _, g := range grants { + switch { + case g.Permission == "READ": + res["X-Amz-Grant-Read"] = append(res["X-Amz-Grant-Read"], "id="+g.Grantee.ID) + case g.Permission == "WRITE": + res["X-Amz-Grant-Write"] = append(res["X-Amz-Grant-Write"], "id="+g.Grantee.ID) + case g.Permission == "READ_ACP": + res["X-Amz-Grant-Read-Acp"] = append(res["X-Amz-Grant-Read-Acp"], "id="+g.Grantee.ID) + case g.Permission == "WRITE_ACP": + res["X-Amz-Grant-Write-Acp"] = append(res["X-Amz-Grant-Write-Acp"], "id="+g.Grantee.ID) + case g.Permission == "FULL_CONTROL": + res["X-Amz-Grant-Full-Control"] = append(res["X-Amz-Grant-Full-Control"], "id="+g.Grantee.ID) + } + } + return res +} diff --git a/api-get-object-acl.go b/api-get-object-acl.go index ea809492a..9a664f189 100644 --- a/api-get-object-acl.go +++ b/api-get-object-acl.go @@ -19,118 +19,9 @@ package minio import ( "context" - "net/http" - "net/url" ) -type accessControlPolicy struct { - Owner struct { - ID string `xml:"ID"` - DisplayName string `xml:"DisplayName"` - } `xml:"Owner"` - AccessControlList struct { - Grant []struct { - Grantee struct { - ID string `xml:"ID"` - DisplayName string `xml:"DisplayName"` - URI string `xml:"URI"` - } `xml:"Grantee"` - Permission string `xml:"Permission"` - } `xml:"Grant"` - } `xml:"AccessControlList"` -} - //GetObjectACL get object ACLs func (c Client) GetObjectACL(bucketName, objectName string) (*ObjectInfo, error) { - - resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ - bucketName: bucketName, - objectName: objectName, - queryValues: url.Values{ - "acl": []string{""}, - }, - }) - if err != nil { - return nil, err - } - defer closeResponse(resp) - - if resp.StatusCode != http.StatusOK { - return nil, httpRespToErrorResponse(resp, bucketName, objectName) - } - - res := &accessControlPolicy{} - - if err := xmlDecoder(resp.Body, res); err != nil { - return nil, err - } - - objInfo, err := c.statObject(context.Background(), bucketName, objectName, StatObjectOptions{}) - if err != nil { - return nil, err - } - - cannedACL := getCannedACL(res) - if cannedACL != "" { - objInfo.Metadata.Add("X-Amz-Acl", cannedACL) - return &objInfo, nil - } - - grantACL := getAmzGrantACL(res) - for k, v := range grantACL { - objInfo.Metadata[k] = v - } - - return &objInfo, nil -} - -func getCannedACL(aCPolicy *accessControlPolicy) string { - grants := aCPolicy.AccessControlList.Grant - - switch { - case len(grants) == 1: - if grants[0].Grantee.URI == "" && grants[0].Permission == "FULL_CONTROL" { - return "private" - } - case len(grants) == 2: - for _, g := range grants { - if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" && g.Permission == "READ" { - return "authenticated-read" - } - if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && g.Permission == "READ" { - return "public-read" - } - if g.Permission == "READ" && g.Grantee.ID == aCPolicy.Owner.ID { - return "bucket-owner-read" - } - } - case len(grants) == 3: - for _, g := range grants { - if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && g.Permission == "WRITE" { - return "public-read-write" - } - } - } - return "" -} - -func getAmzGrantACL(aCPolicy *accessControlPolicy) map[string][]string { - grants := aCPolicy.AccessControlList.Grant - res := map[string][]string{} - - for _, g := range grants { - switch { - case g.Permission == "READ": - res["X-Amz-Grant-Read"] = append(res["X-Amz-Grant-Read"], "id="+g.Grantee.ID) - case g.Permission == "WRITE": - res["X-Amz-Grant-Write"] = append(res["X-Amz-Grant-Write"], "id="+g.Grantee.ID) - case g.Permission == "READ_ACP": - res["X-Amz-Grant-Read-Acp"] = append(res["X-Amz-Grant-Read-Acp"], "id="+g.Grantee.ID) - case g.Permission == "WRITE_ACP": - res["X-Amz-Grant-Write-Acp"] = append(res["X-Amz-Grant-Write-Acp"], "id="+g.Grantee.ID) - case g.Permission == "FULL_CONTROL": - res["X-Amz-Grant-Full-Control"] = append(res["X-Amz-Grant-Full-Control"], "id="+g.Grantee.ID) - } - } - return res + return c.GetObjectACLWithContext(context.Background(), bucketName, objectName) } diff --git a/examples/s3/getobjectacl.go b/examples/s3/getobjectacl.go index f557009b2..b4e720706 100644 --- a/examples/s3/getobjectacl.go +++ b/examples/s3/getobjectacl.go @@ -45,6 +45,12 @@ func main() { log.Fatalln(err) } + //print object owner informations + fmt.Printf(`Object owner: +Display name: %q +ID: %q +`, objectInfo.Owner.DisplayName, objectInfo.Owner.ID) + //print all value header (acl, metadata, standard header value...) for k, v := range objectInfo.Metadata { fmt.Println("key:", k) diff --git a/functional_tests.go b/functional_tests.go index 69ae3b2b0..38d69f72d 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -9528,13 +9528,14 @@ func testFGetObjectWithContext() { } -// Test get object ACLs with GetObjectACL -func testGetObjectACL() { +// Test get object ACLs with GetObjectACLWithContext +func testGetObjectACLWithContext() { // initialize logging params startTime := time.Now() testName := getFuncName() - function := "GetObjectACL(bucketName, objectName)" + function := "GetObjectACLWithContext(ctx, bucketName, objectName)" args := map[string]interface{}{ + "ctx": "", "bucketName": "", "objectName": "", } @@ -9594,26 +9595,30 @@ func testGetObjectACL() { return } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + args["ctx"] = ctx + defer cancel() + // Read the data back - objectInfo, getObjectACLErr := c.GetObjectACL(bucketName, objectName) + objectInfo, getObjectACLErr := c.GetObjectACLWithContext(ctx, bucketName, objectName) if getObjectACLErr == nil { - logError(testName, function, args, startTime, "", "GetObjectACL fail", getObjectACLErr) + logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail", getObjectACLErr) return } s, ok := objectInfo.Metadata["X-Amz-Acl"] if !ok { - logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Acl\"", nil) + logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail unable to find \"X-Amz-Acl\"", nil) return } if len(s) != 1 { - logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" canned acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) + logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail \"X-Amz-Acl\" canned acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) return } if s[0] != "public-read-write" { - logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" expected \"public-read-write\" but got"+fmt.Sprintf("%q", s[0]), nil) + logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail \"X-Amz-Acl\" expected \"public-read-write\" but got"+fmt.Sprintf("%q", s[0]), nil) return } @@ -9636,47 +9641,51 @@ func testGetObjectACL() { return } + ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) + args["ctx"] = ctx + defer cancel() + // Read the data back - objectInfo, getObjectACLErr = c.GetObjectACL(bucketName, objectName) + objectInfo, getObjectACLErr = c.GetObjectACLWithContext(ctx, bucketName, objectName) if getObjectACLErr == nil { - logError(testName, function, args, startTime, "", "GetObjectACL fail", getObjectACLErr) + logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail", getObjectACLErr) return } if len(objectInfo.Metadata) != 3 { - logError(testName, function, args, startTime, "", "GetObjectACL fail expected \"3\" ACLs but got "+fmt.Sprintf(`"%d"`, len(objectInfo.Metadata)), nil) + logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail expected \"3\" ACLs but got "+fmt.Sprintf(`"%d"`, len(objectInfo.Metadata)), nil) return } s, ok = objectInfo.Metadata["X-Amz-Grant-Read"] if !ok { - logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Grant-Read\"", nil) + logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail unable to find \"X-Amz-Grant-Read\"", nil) return } if len(s) != 1 { - logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Read\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) + logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail \"X-Amz-Grant-Read\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) return } if s[0] != "fooread@minio.go" { - logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Read\" acl expected \"fooread@minio.go\" got "+fmt.Sprintf("%q", s), nil) + logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail \"X-Amz-Grant-Read\" acl expected \"fooread@minio.go\" got "+fmt.Sprintf("%q", s), nil) return } s, ok = objectInfo.Metadata["X-Amz-Grant-Write"] if !ok { - logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Grant-Write\"", nil) + logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail unable to find \"X-Amz-Grant-Write\"", nil) return } if len(s) != 1 { - logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Write\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) + logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail \"X-Amz-Grant-Write\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) return } if s[0] != "foowrite@minio.go" { - logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Write\" acl expected \"foowrite@minio.go\" got "+fmt.Sprintf("%q", s), nil) + logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail \"X-Amz-Grant-Write\" acl expected \"foowrite@minio.go\" got "+fmt.Sprintf("%q", s), nil) return } @@ -10146,7 +10155,7 @@ func main() { testFPutObjectWithContext() testFGetObjectWithContext() - testGetObjectACL() + testGetObjectACLWithContext() testPutObjectWithContext() testStorageClassMetadataPutObject() From 351efed7cc2219c9db0cce3713866451ef7d2ba9 Mon Sep 17 00:00:00 2001 From: illi-j Date: Wed, 4 Dec 2019 16:57:52 +0100 Subject: [PATCH 082/215] Add grant information in GetObjectACL (#1190) --- api-datatypes.go | 10 ++++++++++ api-get-object-acl-context.go | 2 ++ examples/s3/getobjectacl.go | 10 ++++++++++ 3 files changed, 22 insertions(+) diff --git a/api-datatypes.go b/api-datatypes.go index 4e54f3415..e81a3bd41 100644 --- a/api-datatypes.go +++ b/api-datatypes.go @@ -88,6 +88,16 @@ type ObjectInfo struct { ID string `json:"id"` } `json:"owner"` + // ACL grant. + Grant []struct { + Grantee struct { + ID string `xml:"ID"` + DisplayName string `xml:"DisplayName"` + URI string `xml:"URI"` + } `xml:"Grantee"` + Permission string `xml:"Permission"` + } `xml:"Grant"` + // The class of storage used to store the object. StorageClass string `json:"storageClass"` diff --git a/api-get-object-acl-context.go b/api-get-object-acl-context.go index 24d87671c..f49984f68 100644 --- a/api-get-object-acl-context.go +++ b/api-get-object-acl-context.go @@ -72,6 +72,8 @@ func (c Client) GetObjectACLWithContext(ctx context.Context, bucketName, objectN objInfo.Owner.DisplayName = res.Owner.DisplayName objInfo.Owner.ID = res.Owner.ID + objInfo.Grant = append(objInfo.Grant, res.AccessControlList.Grant...) + cannedACL := getCannedACL(res) if cannedACL != "" { objInfo.Metadata.Add("X-Amz-Acl", cannedACL) diff --git a/examples/s3/getobjectacl.go b/examples/s3/getobjectacl.go index b4e720706..3b02a918d 100644 --- a/examples/s3/getobjectacl.go +++ b/examples/s3/getobjectacl.go @@ -51,6 +51,16 @@ Display name: %q ID: %q `, objectInfo.Owner.DisplayName, objectInfo.Owner.ID) + //print object grant informations + for _, g := range objectInfo.Grant { + fmt.Printf(`Object grant: + - Display name: %q + - ID: %q + - URI: %q + - Permission: %q +`, g.Grantee.DisplayName, g.Grantee.ID, g.Grantee.URI, g.Permission) + } + //print all value header (acl, metadata, standard header value...) for k, v := range objectInfo.Metadata { fmt.Println("key:", k) From 8e3928466d18a04caf35a76a0f4a41d65ab1c9fc Mon Sep 17 00:00:00 2001 From: Daniel Valdivia Date: Thu, 5 Dec 2019 12:09:39 -0800 Subject: [PATCH 083/215] Add Bucket APIs with context support (#1192) --- api-list.go | 17 ++++++++- api-put-bucket.go | 91 ++++++++++++++++++++++++++++++++++----------- api-stat.go | 16 +++++++- core.go | 8 +++- functional_tests.go | 45 +++++++++++++++++++++- 5 files changed, 150 insertions(+), 27 deletions(-) diff --git a/api-list.go b/api-list.go index 09c1565cb..3329d35fb 100644 --- a/api-list.go +++ b/api-list.go @@ -39,8 +39,23 @@ import ( // } // func (c Client) ListBuckets() ([]BucketInfo, error) { + return c.ListBucketsWithContext(context.Background()) +} + +// ListBucketsWithContext list all buckets owned by this authenticated user, +// accepts a context for facilitate cancellation. +// +// This call requires explicit authentication, no anonymous requests are +// allowed for listing buckets. +// +// api := client.New(....) +// for message := range api.ListBucketsWithContext(context.Background()) { +// fmt.Println(message) +// } +// +func (c Client) ListBucketsWithContext(ctx context.Context) ([]BucketInfo, error) { // Execute GET on service. - resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{contentSHA256Hex: emptySHA256Hex}) + resp, err := c.executeMethod(ctx, "GET", requestMetadata{contentSHA256Hex: emptySHA256Hex}) defer closeResponse(resp) if err != nil { return nil, err diff --git a/api-put-bucket.go b/api-put-bucket.go index 0041ce179..68371986a 100644 --- a/api-put-bucket.go +++ b/api-put-bucket.go @@ -31,7 +31,7 @@ import ( /// Bucket operations -func (c Client) makeBucket(bucketName string, location string, objectLockEnabled bool) (err error) { +func (c Client) makeBucket(ctx context.Context, bucketName string, location string, objectLockEnabled bool) (err error) { defer func() { // Save the location into cache on a successful makeBucket response. if err == nil { @@ -81,7 +81,7 @@ func (c Client) makeBucket(bucketName string, location string, objectLockEnabled } // Execute PUT to create a new bucket. - resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) + resp, err := c.executeMethod(ctx, "PUT", reqMetadata) defer closeResponse(resp) if err != nil { return err @@ -105,7 +105,18 @@ func (c Client) makeBucket(bucketName string, location string, objectLockEnabled // For Amazon S3 for more supported regions - http://docs.aws.amazon.com/general/latest/gr/rande.html // For Google Cloud Storage for more supported regions - https://cloud.google.com/storage/docs/bucket-locations func (c Client) MakeBucket(bucketName string, location string) (err error) { - return c.makeBucket(bucketName, location, false) + return c.MakeBucketWithContext(context.Background(), bucketName, location) +} + +// MakeBucketWithContext creates a new bucket with bucketName with a context to control cancellations and timeouts. +// +// Location is an optional argument, by default all buckets are +// created in US Standard Region. +// +// For Amazon S3 for more supported regions - http://docs.aws.amazon.com/general/latest/gr/rande.html +// For Google Cloud Storage for more supported regions - https://cloud.google.com/storage/docs/bucket-locations +func (c Client) MakeBucketWithContext(ctx context.Context, bucketName string, location string) (err error) { + return c.makeBucket(ctx, bucketName, location, false) } // MakeBucketWithObjectLock creates a object lock enabled new bucket with bucketName. @@ -116,11 +127,28 @@ func (c Client) MakeBucket(bucketName string, location string) (err error) { // For Amazon S3 for more supported regions - http://docs.aws.amazon.com/general/latest/gr/rande.html // For Google Cloud Storage for more supported regions - https://cloud.google.com/storage/docs/bucket-locations func (c Client) MakeBucketWithObjectLock(bucketName string, location string) (err error) { - return c.makeBucket(bucketName, location, true) + return c.MakeBucketWithObjectLockWithContext(context.Background(), bucketName, location) +} + +// MakeBucketWithObjectLockWithContext creates a object lock enabled new bucket with bucketName with a context to +// control cancellations and timeouts. +// +// Location is an optional argument, by default all buckets are +// created in US Standard Region. +// +// For Amazon S3 for more supported regions - http://docs.aws.amazon.com/general/latest/gr/rande.html +// For Google Cloud Storage for more supported regions - https://cloud.google.com/storage/docs/bucket-locations +func (c Client) MakeBucketWithObjectLockWithContext(ctx context.Context, bucketName string, location string) (err error) { + return c.makeBucket(ctx, bucketName, location, true) } // SetBucketPolicy set the access permissions on an existing bucket. func (c Client) SetBucketPolicy(bucketName, policy string) error { + return c.SetBucketPolicyWithContext(context.Background(), bucketName, policy) +} + +// SetBucketPolicyWithContext set the access permissions on an existing bucket. +func (c Client) SetBucketPolicyWithContext(ctx context.Context, bucketName, policy string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -128,15 +156,15 @@ func (c Client) SetBucketPolicy(bucketName, policy string) error { // If policy is empty then delete the bucket policy. if policy == "" { - return c.removeBucketPolicy(bucketName) + return c.removeBucketPolicy(ctx, bucketName) } // Save the updated policies. - return c.putBucketPolicy(bucketName, policy) + return c.putBucketPolicy(ctx, bucketName, policy) } // Saves a new bucket policy. -func (c Client) putBucketPolicy(bucketName, policy string) error { +func (c Client) putBucketPolicy(ctx context.Context, bucketName, policy string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -162,7 +190,7 @@ func (c Client) putBucketPolicy(bucketName, policy string) error { } // Execute PUT to upload a new bucket policy. - resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) + resp, err := c.executeMethod(ctx, "PUT", reqMetadata) defer closeResponse(resp) if err != nil { return err @@ -176,7 +204,7 @@ func (c Client) putBucketPolicy(bucketName, policy string) error { } // Removes all policies on a bucket. -func (c Client) removeBucketPolicy(bucketName string) error { +func (c Client) removeBucketPolicy(ctx context.Context, bucketName string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -187,7 +215,7 @@ func (c Client) removeBucketPolicy(bucketName string) error { urlValues.Set("policy", "") // Execute DELETE on objectName. - resp, err := c.executeMethod(context.Background(), "DELETE", requestMetadata{ + resp, err := c.executeMethod(ctx, "DELETE", requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, @@ -201,6 +229,11 @@ func (c Client) removeBucketPolicy(bucketName string) error { // SetBucketLifecycle set the lifecycle on an existing bucket. func (c Client) SetBucketLifecycle(bucketName, lifecycle string) error { + return c.SetBucketLifecycleWithContext(context.Background(), bucketName, lifecycle) +} + +// SetBucketLifecycleWithContext set the lifecycle on an existing bucket with a context to control cancellations and timeouts. +func (c Client) SetBucketLifecycleWithContext(ctx context.Context, bucketName, lifecycle string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -208,15 +241,15 @@ func (c Client) SetBucketLifecycle(bucketName, lifecycle string) error { // If lifecycle is empty then delete it. if lifecycle == "" { - return c.removeBucketLifecycle(bucketName) + return c.removeBucketLifecycle(ctx, bucketName) } // Save the updated lifecycle. - return c.putBucketLifecycle(bucketName, lifecycle) + return c.putBucketLifecycle(ctx, bucketName, lifecycle) } // Saves a new bucket lifecycle. -func (c Client) putBucketLifecycle(bucketName, lifecycle string) error { +func (c Client) putBucketLifecycle(ctx context.Context, bucketName, lifecycle string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -243,7 +276,7 @@ func (c Client) putBucketLifecycle(bucketName, lifecycle string) error { } // Execute PUT to upload a new bucket lifecycle. - resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) + resp, err := c.executeMethod(ctx, "PUT", reqMetadata) defer closeResponse(resp) if err != nil { return err @@ -257,7 +290,7 @@ func (c Client) putBucketLifecycle(bucketName, lifecycle string) error { } // Remove lifecycle from a bucket. -func (c Client) removeBucketLifecycle(bucketName string) error { +func (c Client) removeBucketLifecycle(ctx context.Context, bucketName string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -268,7 +301,7 @@ func (c Client) removeBucketLifecycle(bucketName string) error { urlValues.Set("lifecycle", "") // Execute DELETE on objectName. - resp, err := c.executeMethod(context.Background(), "DELETE", requestMetadata{ + resp, err := c.executeMethod(ctx, "DELETE", requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, @@ -282,6 +315,12 @@ func (c Client) removeBucketLifecycle(bucketName string) error { // SetBucketNotification saves a new bucket notification. func (c Client) SetBucketNotification(bucketName string, bucketNotification BucketNotification) error { + return c.SetBucketNotificationWithContext(context.Background(), bucketName, bucketNotification) +} + +// SetBucketNotificationWithContext saves a new bucket notification with a context to control cancellations +// and timeouts. +func (c Client) SetBucketNotificationWithContext(ctx context.Context, bucketName string, bucketNotification BucketNotification) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -308,7 +347,7 @@ func (c Client) SetBucketNotification(bucketName string, bucketNotification Buck } // Execute PUT to upload a new bucket notification. - resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) + resp, err := c.executeMethod(ctx, "PUT", reqMetadata) defer closeResponse(resp) if err != nil { return err @@ -338,7 +377,7 @@ var ( versionDisableConfigSHA256 = sum256Hex(versionDisableConfig) ) -func (c Client) setVersioning(bucketName string, config []byte, length int64, md5sum, sha256sum string) error { +func (c Client) setVersioning(ctx context.Context, bucketName string, config []byte, length int64, md5sum, sha256sum string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -359,7 +398,7 @@ func (c Client) setVersioning(bucketName string, config []byte, length int64, md } // Execute PUT to set a bucket versioning. - resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) + resp, err := c.executeMethod(ctx, "PUT", reqMetadata) defer closeResponse(resp) if err != nil { return err @@ -374,10 +413,20 @@ func (c Client) setVersioning(bucketName string, config []byte, length int64, md // EnableVersioning - Enable object versioning in given bucket. func (c Client) EnableVersioning(bucketName string) error { - return c.setVersioning(bucketName, versionEnableConfig, versionEnableConfigLen, versionEnableConfigMD5Sum, versionEnableConfigSHA256) + return c.EnableVersioningWithContext(context.Background(), bucketName) +} + +// EnableVersioningWithContext - Enable object versioning in given bucket with a context to control cancellations and timeouts. +func (c Client) EnableVersioningWithContext(ctx context.Context, bucketName string) error { + return c.setVersioning(ctx, bucketName, versionEnableConfig, versionEnableConfigLen, versionEnableConfigMD5Sum, versionEnableConfigSHA256) } // DisableVersioning - Disable object versioning in given bucket. func (c Client) DisableVersioning(bucketName string) error { - return c.setVersioning(bucketName, versionDisableConfig, versionDisableConfigLen, versionDisableConfigMD5Sum, versionDisableConfigSHA256) + return c.DisableVersioningWithContext(context.Background(), bucketName) +} + +// DisableVersioningWithContext - Disable object versioning in given bucket with a context to control cancellations and timeouts. +func (c Client) DisableVersioningWithContext(ctx context.Context, bucketName string) error { + return c.setVersioning(ctx, bucketName, versionDisableConfig, versionDisableConfigLen, versionDisableConfigMD5Sum, versionDisableConfigSHA256) } diff --git a/api-stat.go b/api-stat.go index 5cc40e757..fe246e318 100644 --- a/api-stat.go +++ b/api-stat.go @@ -29,13 +29,19 @@ import ( // BucketExists verify if bucket exists and you have permission to access it. func (c Client) BucketExists(bucketName string) (bool, error) { + return c.BucketExistsWithContext(context.Background(), bucketName) +} + +// BucketExistsWithContext verify if bucket exists and you have permission to access it. Allows for a Context to +// control cancellations and timeouts. +func (c Client) BucketExistsWithContext(ctx context.Context, bucketName string) (bool, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return false, err } // Execute HEAD on bucketName. - resp, err := c.executeMethod(context.Background(), "HEAD", requestMetadata{ + resp, err := c.executeMethod(ctx, "HEAD", requestMetadata{ bucketName: bucketName, contentSHA256Hex: emptySHA256Hex, }) @@ -91,6 +97,12 @@ func extractObjMetadata(header http.Header) http.Header { // StatObject verifies if object exists and you have permission to access. func (c Client) StatObject(bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { + return c.StatObjectWithContext(context.Background(), bucketName, objectName, opts) +} + +// StatObjectWithContext verifies if object exists and you have permission to access with a context to control +// cancellations and timeouts. +func (c Client) StatObjectWithContext(ctx context.Context, bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ObjectInfo{}, err @@ -98,7 +110,7 @@ func (c Client) StatObject(bucketName, objectName string, opts StatObjectOptions if err := s3utils.CheckValidObjectName(objectName); err != nil { return ObjectInfo{}, err } - return c.statObject(context.Background(), bucketName, objectName, opts) + return c.statObject(ctx, bucketName, objectName, opts) } // Lower level API for statObject supporting pre-conditions and range headers. diff --git a/core.go b/core.go index 0d43ae0f5..277210a8d 100644 --- a/core.go +++ b/core.go @@ -171,7 +171,13 @@ func (c Core) GetBucketPolicy(bucket string) (string, error) { // PutBucketPolicy - applies a new bucket access policy for a given bucket. func (c Core) PutBucketPolicy(bucket, bucketPolicy string) error { - return c.putBucketPolicy(bucket, bucketPolicy) + return c.PutBucketPolicyWithContext(context.Background(), bucket, bucketPolicy) +} + +// PutBucketPolicyWithContext - applies a new bucket access policy for a given bucket with a context to control +// cancellations and timeouts. +func (c Core) PutBucketPolicyWithContext(ctx context.Context, bucket, bucketPolicy string) error { + return c.putBucketPolicy(ctx, bucket, bucketPolicy) } // GetObjectWithContext is a lower level API implemented to support reading diff --git a/functional_tests.go b/functional_tests.go index 38d69f72d..fb66f06fe 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -39,7 +39,7 @@ import ( "strings" "time" - humanize "github.com/dustin/go-humanize" + "github.com/dustin/go-humanize" log "github.com/sirupsen/logrus" "github.com/minio/minio-go/v6" @@ -4207,6 +4207,23 @@ func testFunctional() { return } + // Verify if bucket exits and you have access with context. + function = "BucketExistsWithContext(ctx, bucketName)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + } + exists, err = c.BucketExistsWithContext(context.Background(), bucketName) + + if err != nil { + logError(testName, function, args, startTime, "", "BucketExistsWithContext failed", err) + return + } + if !exists { + logError(testName, function, args, startTime, "", "Could not find the bucket", err) + return + } + // Asserting the default bucket policy. function = "GetBucketPolicy(bucketName)" functionAll += ", " + function @@ -4321,6 +4338,21 @@ func testFunctional() { return } + // List all buckets with context. + function = "ListBucketsWithContext()" + functionAll += ", " + function + args = nil + buckets, err = c.ListBucketsWithContext(context.Background()) + + if len(buckets) == 0 { + logError(testName, function, args, startTime, "", "Found bucket list to be empty", err) + return + } + if err != nil { + logError(testName, function, args, startTime, "", "ListBucketsWithContext failed", err) + return + } + // Verify if previously created bucket is listed in list buckets. bucketFound := false for _, bucket := range buckets { @@ -5287,6 +5319,16 @@ func testFPutObjectV2() { return } + rGTar, err = c.StatObjectWithContext(context.Background(), bucketName, objectName+"-GTar", minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject failed", err) + return + } + if rGTar.ContentType != "application/x-gtar" && rGTar.ContentType != "application/octet-stream" { + logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/x-gtar , got "+rGTar.ContentType, err) + return + } + // Delete all objects and buckets if err = cleanupBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "Cleanup failed", err) @@ -10025,7 +10067,6 @@ func testListObjects() { name string storageClass string }{ - // \x17 is a forbidden character in a xml document {"foo\x17bar", "STANDARD"}, // Special characters From f0215220d843a3601d5e32d0b7eda69667c3d96c Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Sat, 7 Dec 2019 01:19:27 +0000 Subject: [PATCH 084/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index f0978b633..25cc3d676 100644 --- a/api.go +++ b/api.go @@ -104,7 +104,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.44" + libraryVersion = "v6.0.45" ) // User Agent should always following the below style. From b1f012abb4341666efc6338e6e6fe0b9c585b3d1 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 9 Dec 2019 00:39:23 -0800 Subject: [PATCH 085/215] Trim all etags with single function (#1197) Add retry code to retry more errors --- api-get-object.go | 3 +-- api-list.go | 5 ++-- api-put-object-multipart.go | 3 +-- api-put-object-streaming.go | 3 +-- api-stat.go | 3 +-- api.go | 7 +++++ retry.go | 53 ++++++++++++++++++++++--------------- utils.go | 5 ++++ 8 files changed, 50 insertions(+), 32 deletions(-) diff --git a/api-get-object.go b/api-get-object.go index 136782806..5654a602f 100644 --- a/api-get-object.go +++ b/api-get-object.go @@ -622,8 +622,7 @@ func (c Client) getObject(ctx context.Context, bucketName, objectName string, op } // Trim off the odd double quotes from ETag in the beginning and end. - md5sum := strings.TrimPrefix(resp.Header.Get("ETag"), "\"") - md5sum = strings.TrimSuffix(md5sum, "\"") + md5sum := trimEtag(resp.Header.Get("ETag")) // Parse the date. date, err := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified")) diff --git a/api-list.go b/api-list.go index 3329d35fb..a82c75cf8 100644 --- a/api-list.go +++ b/api-list.go @@ -23,7 +23,6 @@ import ( "fmt" "net/http" "net/url" - "strings" "github.com/minio/minio-go/v6/pkg/s3utils" ) @@ -155,6 +154,7 @@ func (c Client) listObjectsV2(bucketName, objectPrefix string, recursive, metada // If contents are available loop through and send over channel. for _, object := range result.Contents { + object.ETag = trimEtag(object.ETag) select { // Send object content. case objectStatCh <- object: @@ -724,8 +724,7 @@ func (c Client) listObjectParts(bucketName, objectName, uploadID string) (partsI // Append to parts info. for _, part := range listObjPartsResult.ObjectParts { // Trim off the odd double quotes from ETag in the beginning and end. - part.ETag = strings.TrimPrefix(part.ETag, "\"") - part.ETag = strings.TrimSuffix(part.ETag, "\"") + part.ETag = trimEtag(part.ETag) partsInfo[part.PartNumber] = part } // Keep part number marker, for the next iteration. diff --git a/api-put-object-multipart.go b/api-put-object-multipart.go index ab284f986..3b39ed2bd 100644 --- a/api-put-object-multipart.go +++ b/api-put-object-multipart.go @@ -294,8 +294,7 @@ func (c Client) uploadPart(ctx context.Context, bucketName, objectName, uploadID objPart.Size = size objPart.PartNumber = partNumber // Trim off the odd double quotes from ETag in the beginning and end. - objPart.ETag = strings.TrimPrefix(resp.Header.Get("ETag"), "\"") - objPart.ETag = strings.TrimSuffix(objPart.ETag, "\"") + objPart.ETag = trimEtag(resp.Header.Get("ETag")) return objPart, nil } diff --git a/api-put-object-streaming.go b/api-put-object-streaming.go index 0d3f2455a..46b93e602 100644 --- a/api-put-object-streaming.go +++ b/api-put-object-streaming.go @@ -407,8 +407,7 @@ func (c Client) putObjectDo(ctx context.Context, bucketName, objectName string, var objInfo ObjectInfo // Trim off the odd double quotes from ETag in the beginning and end. - objInfo.ETag = strings.TrimPrefix(resp.Header.Get("ETag"), "\"") - objInfo.ETag = strings.TrimSuffix(objInfo.ETag, "\"") + objInfo.ETag = trimEtag(resp.Header.Get("ETag")) // A success here means data was written to server successfully. objInfo.Size = size diff --git a/api-stat.go b/api-stat.go index fe246e318..525253a3a 100644 --- a/api-stat.go +++ b/api-stat.go @@ -141,8 +141,7 @@ func (c Client) statObject(ctx context.Context, bucketName, objectName string, o } // Trim off the odd double quotes from ETag in the beginning and end. - md5sum := strings.TrimPrefix(resp.Header.Get("ETag"), "\"") - md5sum = strings.TrimSuffix(md5sum, "\"") + md5sum := trimEtag(resp.Header.Get("ETag")) // Parse content length is exists var size int64 = -1 diff --git a/api.go b/api.go index 25cc3d676..4c5fb7ecc 100644 --- a/api.go +++ b/api.go @@ -560,6 +560,13 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque var bodySeeker io.Seeker // Extracted seeker from io.Reader. var reqRetry = MaxRetry // Indicates how many times we can retry the request + defer func() { + if err != nil { + // close idle connections before returning, upon error. + c.httpClient.CloseIdleConnections() + } + }() + if metadata.contentBody != nil { // Check if body is seekable then it is retryable. bodySeeker, isRetryable = metadata.contentBody.(io.Seeker) diff --git a/retry.go b/retry.go index 2c608bab5..ef2db53f8 100644 --- a/retry.go +++ b/retry.go @@ -91,31 +91,42 @@ func isHTTPReqErrorRetryable(err error) bool { if err == nil { return false } - switch e := err.(type) { - case *url.Error: - switch e.Err.(type) { - case *net.DNSError, *net.OpError, net.UnknownNetworkError: - return true + // We need to figure if the error either a timeout + // or a non-temporary error. + e, ok := err.(net.Error) + if ok { + urlErr, ok := e.(*url.Error) + if ok { + switch urlErr.Err.(type) { + case *net.DNSError, *net.OpError, net.UnknownNetworkError: + return true + } } - if strings.Contains(err.Error(), "Connection closed by foreign host") { - return true - } else if strings.Contains(err.Error(), "net/http: TLS handshake timeout") { - // If error is - tlsHandshakeTimeoutError, retry. - return true - } else if strings.Contains(err.Error(), "i/o timeout") { - // If error is - tcp timeoutError, retry. - return true - } else if strings.Contains(err.Error(), "connection timed out") { - // If err is a net.Dial timeout, retry. - return true - } else if strings.Contains(err.Error(), "net/http: HTTP/1.x transport connection broken") { - // If error is transport connection broken, retry. - return true - } else if strings.Contains(err.Error(), "net/http: timeout awaiting response headers") { - // Retry errors due to server not sending the response before timeout + if e.Timeout() { return true } } + if strings.Contains(err.Error(), "Connection closed by foreign host") { + return true + } else if strings.Contains(err.Error(), "net/http: TLS handshake timeout") { + // If error is - tlsHandshakeTimeoutError, retry. + return true + } else if strings.Contains(err.Error(), "i/o timeout") { + // If error is - tcp timeoutError, retry. + return true + } else if strings.Contains(err.Error(), "connection timed out") { + // If err is a net.Dial timeout, retry. + return true + } else if strings.Contains(err.Error(), "net/http: HTTP/1.x transport connection broken") { + // If error is transport connection broken, retry. + return true + } else if strings.Contains(err.Error(), "net/http: timeout awaiting response headers") { + // Retry errors due to server not sending the response before timeout + return true + } else if strings.Contains(err.Error(), "connection reset by peer") { + // Retry errors due to connection reset by peer. + return true + } return false } diff --git a/utils.go b/utils.go index d24cfb5c7..5b324547d 100644 --- a/utils.go +++ b/utils.go @@ -36,6 +36,11 @@ import ( "github.com/minio/minio-go/v6/pkg/s3utils" ) +func trimEtag(etag string) string { + etag = strings.TrimPrefix(etag, "\"") + return strings.TrimSuffix(etag, "\"") +} + // xmlDecoder provide decoded value in xml. func xmlDecoder(body io.Reader, v interface{}) error { d := xml.NewDecoder(body) From 97b0a62ed75d2f3d3d4e2deed9c7d97066149884 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 9 Dec 2019 01:27:16 -0800 Subject: [PATCH 086/215] Retry for all network errors regardless (#1198) --- api.go | 7 +------ retry.go | 48 ------------------------------------------------ 2 files changed, 1 insertion(+), 54 deletions(-) diff --git a/api.go b/api.go index 4c5fb7ecc..9b6b260d0 100644 --- a/api.go +++ b/api.go @@ -627,12 +627,7 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque // Initiate the request. res, err = c.do(req) if err != nil { - // For supported http requests errors verify. - if isHTTPReqErrorRetryable(err) { - continue // Retry. - } - // For other errors, return here no need to retry. - return nil, err + continue } // For any known successful http status, return quickly. diff --git a/retry.go b/retry.go index ef2db53f8..f9c1095c0 100644 --- a/retry.go +++ b/retry.go @@ -18,10 +18,7 @@ package minio import ( - "net" "net/http" - "net/url" - "strings" "time" ) @@ -85,51 +82,6 @@ func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duratio return attemptCh } -// isHTTPReqErrorRetryable - is http requests error retryable, such -// as i/o timeout, connection broken etc.. -func isHTTPReqErrorRetryable(err error) bool { - if err == nil { - return false - } - // We need to figure if the error either a timeout - // or a non-temporary error. - e, ok := err.(net.Error) - if ok { - urlErr, ok := e.(*url.Error) - if ok { - switch urlErr.Err.(type) { - case *net.DNSError, *net.OpError, net.UnknownNetworkError: - return true - } - } - if e.Timeout() { - return true - } - } - if strings.Contains(err.Error(), "Connection closed by foreign host") { - return true - } else if strings.Contains(err.Error(), "net/http: TLS handshake timeout") { - // If error is - tlsHandshakeTimeoutError, retry. - return true - } else if strings.Contains(err.Error(), "i/o timeout") { - // If error is - tcp timeoutError, retry. - return true - } else if strings.Contains(err.Error(), "connection timed out") { - // If err is a net.Dial timeout, retry. - return true - } else if strings.Contains(err.Error(), "net/http: HTTP/1.x transport connection broken") { - // If error is transport connection broken, retry. - return true - } else if strings.Contains(err.Error(), "net/http: timeout awaiting response headers") { - // Retry errors due to server not sending the response before timeout - return true - } else if strings.Contains(err.Error(), "connection reset by peer") { - // Retry errors due to connection reset by peer. - return true - } - return false -} - // List of AWS S3 error codes which are retryable. var retryableS3Codes = map[string]struct{}{ "RequestError": {}, From fc36ccece739689492aa05506ee0473c1257c712 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 12 Dec 2019 10:21:50 -0800 Subject: [PATCH 087/215] Add ListenBucketNotificationV2 (#1199) --- api-notification.go | 98 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/api-notification.go b/api-notification.go index 0480c21eb..5bb846877 100644 --- a/api-notification.go +++ b/api-notification.go @@ -229,3 +229,101 @@ func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, even // Returns the notification info channel, for caller to start reading from. return notificationInfoCh } + +// ListenBucketNotificationV2 - listen on bucket notifications. +func (c Client) ListenBucketNotificationV2(bucketName, prefix, suffix string, events []string, doneCh <-chan struct{}) <-chan NotificationInfo { + notificationInfoCh := make(chan NotificationInfo, 1) + // Only success, start a routine to start reading line by line. + go func(notificationInfoCh chan<- NotificationInfo) { + defer close(notificationInfoCh) + + // Validate the bucket name. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + notificationInfoCh <- NotificationInfo{ + Err: err, + } + return + } + + // Check ARN partition to verify if listening bucket is supported + if s3utils.IsAmazonEndpoint(*c.endpointURL) || s3utils.IsGoogleEndpoint(*c.endpointURL) { + notificationInfoCh <- NotificationInfo{ + Err: ErrAPINotSupported("Listening for bucket notification is specific only to `minio` server endpoints"), + } + return + } + + // Continuously run and listen on bucket notification. + // Create a done channel to control 'ListObjects' go routine. + retryDoneCh := make(chan struct{}, 1) + + // Indicate to our routine to exit cleanly upon return. + defer close(retryDoneCh) + + // Prepare urlValues to pass into the request on every loop + urlValues := make(url.Values) + urlValues.Set("type", "2") + urlValues.Set("prefix", prefix) + urlValues.Set("suffix", suffix) + urlValues["events"] = events + + // Wait on the jitter retry loop. + for range c.newRetryTimerContinous(time.Second, time.Second*30, MaxJitter, retryDoneCh) { + // Execute GET on bucket to list objects. + resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + if err != nil { + notificationInfoCh <- NotificationInfo{ + Err: err, + } + return + } + + // Validate http response, upon error return quickly. + if resp.StatusCode != http.StatusOK { + errResponse := httpRespToErrorResponse(resp, bucketName, "") + notificationInfoCh <- NotificationInfo{ + Err: errResponse, + } + return + } + + // Initialize a new bufio scanner, to read line by line. + bio := bufio.NewScanner(resp.Body) + + // Unmarshal each line, returns marshalled values. + for bio.Scan() { + var notificationInfo NotificationInfo + if err = json.Unmarshal(bio.Bytes(), ¬ificationInfo); err != nil { + // Unexpected error during json unmarshal, send + // the error to caller for actionable as needed. + notificationInfoCh <- NotificationInfo{ + Err: err, + } + closeResponse(resp) + continue + } + // Send notificationInfo + select { + case notificationInfoCh <- notificationInfo: + case <-doneCh: + closeResponse(resp) + return + } + } + if err = bio.Err(); err != nil { + notificationInfoCh <- NotificationInfo{ + Err: err, + } + } + // Close current connection before looping further. + closeResponse(resp) + } + }(notificationInfoCh) + + // Returns the notification info channel, for caller to start reading from. + return notificationInfoCh +} From a5786a9c2a5b61012d698b18d6ecf308337e8d86 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 13 Dec 2019 11:31:29 -0800 Subject: [PATCH 088/215] Remove ListenBucketNotificationV2 (#1204) Server moves to purge old API, functionality hasn't changed we don't need to change anything on client side so removing this API as its redundant. --- api-notification.go | 98 --------------------------------------------- 1 file changed, 98 deletions(-) diff --git a/api-notification.go b/api-notification.go index 5bb846877..0480c21eb 100644 --- a/api-notification.go +++ b/api-notification.go @@ -229,101 +229,3 @@ func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, even // Returns the notification info channel, for caller to start reading from. return notificationInfoCh } - -// ListenBucketNotificationV2 - listen on bucket notifications. -func (c Client) ListenBucketNotificationV2(bucketName, prefix, suffix string, events []string, doneCh <-chan struct{}) <-chan NotificationInfo { - notificationInfoCh := make(chan NotificationInfo, 1) - // Only success, start a routine to start reading line by line. - go func(notificationInfoCh chan<- NotificationInfo) { - defer close(notificationInfoCh) - - // Validate the bucket name. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - notificationInfoCh <- NotificationInfo{ - Err: err, - } - return - } - - // Check ARN partition to verify if listening bucket is supported - if s3utils.IsAmazonEndpoint(*c.endpointURL) || s3utils.IsGoogleEndpoint(*c.endpointURL) { - notificationInfoCh <- NotificationInfo{ - Err: ErrAPINotSupported("Listening for bucket notification is specific only to `minio` server endpoints"), - } - return - } - - // Continuously run and listen on bucket notification. - // Create a done channel to control 'ListObjects' go routine. - retryDoneCh := make(chan struct{}, 1) - - // Indicate to our routine to exit cleanly upon return. - defer close(retryDoneCh) - - // Prepare urlValues to pass into the request on every loop - urlValues := make(url.Values) - urlValues.Set("type", "2") - urlValues.Set("prefix", prefix) - urlValues.Set("suffix", suffix) - urlValues["events"] = events - - // Wait on the jitter retry loop. - for range c.newRetryTimerContinous(time.Second, time.Second*30, MaxJitter, retryDoneCh) { - // Execute GET on bucket to list objects. - resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - contentSHA256Hex: emptySHA256Hex, - }) - if err != nil { - notificationInfoCh <- NotificationInfo{ - Err: err, - } - return - } - - // Validate http response, upon error return quickly. - if resp.StatusCode != http.StatusOK { - errResponse := httpRespToErrorResponse(resp, bucketName, "") - notificationInfoCh <- NotificationInfo{ - Err: errResponse, - } - return - } - - // Initialize a new bufio scanner, to read line by line. - bio := bufio.NewScanner(resp.Body) - - // Unmarshal each line, returns marshalled values. - for bio.Scan() { - var notificationInfo NotificationInfo - if err = json.Unmarshal(bio.Bytes(), ¬ificationInfo); err != nil { - // Unexpected error during json unmarshal, send - // the error to caller for actionable as needed. - notificationInfoCh <- NotificationInfo{ - Err: err, - } - closeResponse(resp) - continue - } - // Send notificationInfo - select { - case notificationInfoCh <- notificationInfo: - case <-doneCh: - closeResponse(resp) - return - } - } - if err = bio.Err(); err != nil { - notificationInfoCh <- NotificationInfo{ - Err: err, - } - } - // Close current connection before looping further. - closeResponse(resp) - } - }(notificationInfoCh) - - // Returns the notification info channel, for caller to start reading from. - return notificationInfoCh -} From cbd7d666463783f667400c2a6cda3721de913ffc Mon Sep 17 00:00:00 2001 From: Ashish Kumar Sinha Date: Thu, 2 Jan 2020 09:15:23 +0530 Subject: [PATCH 089/215] Add funcion to set policy parameter (#1200) --- post-policy.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/post-policy.go b/post-policy.go index f9250b2a8..8b0bf092d 100644 --- a/post-policy.go +++ b/post-policy.go @@ -131,6 +131,27 @@ func (p *PostPolicy) SetBucket(bucketName string) error { return nil } +// SetCondition - Sets condition for credentials, date and algorithm +func (p *PostPolicy) SetCondition(matchType, condition, value string) error { + if strings.TrimSpace(value) == "" || value == "" { + return ErrInvalidArgument("No value specified for condition") + } + + policyCond := policyCondition{ + matchType: matchType, + condition: "$" + condition, + value: value, + } + if condition == "X-Amz-Credential" || condition == "X-Amz-Date" || condition == "X-Amz-Algorithm" { + if err := p.addNewPolicy(policyCond); err != nil { + return err + } + p.formData[condition] = value + return nil + } + return ErrInvalidArgument("Invalid condition in policy") +} + // SetContentType - Sets content-type of the object for this policy // based upload. func (p *PostPolicy) SetContentType(contentType string) error { From a505b7da22790696cd192361e86499786c47924f Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 6 Jan 2020 16:34:30 -0800 Subject: [PATCH 090/215] fix: AWS Snowball returns 501 handle it (#1208) --- bucket-cache.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bucket-cache.go b/bucket-cache.go index 7ba6cbb57..db8e8e6ce 100644 --- a/bucket-cache.go +++ b/bucket-cache.go @@ -125,6 +125,8 @@ func processBucketLocationResponse(resp *http.Response, bucketName string) (buck // request. Move forward and let the top level callers // succeed if possible based on their policy. switch errResp.Code { + case "NotImplemented": + fallthrough case "AuthorizationHeaderMalformed": fallthrough case "InvalidRegion": From a544e9f394b0e3e9a6f2b97da1084dc3a559b4b0 Mon Sep 17 00:00:00 2001 From: Arran Walker Date: Tue, 7 Jan 2020 11:18:05 +0000 Subject: [PATCH 091/215] Credentials: Support assuming role via WebIdentityTokenFile (#1183) This supports the new AWS_WEB_IDENTITY_TOKEN_FILE and AWS_ROLE_ARN environment variables, that allow exchanging OIDC tokens given to pods in EKS for access tokens. Fixes #1156 --- pkg/credentials/iam_aws.go | 105 +++++++++++++++++----------- pkg/credentials/iam_aws_test.go | 79 +++++++++++++++++++++ pkg/credentials/sts_web_identity.go | 24 ++++++- 3 files changed, 165 insertions(+), 43 deletions(-) diff --git a/pkg/credentials/iam_aws.go b/pkg/credentials/iam_aws.go index 34c593fdf..6c6be6659 100644 --- a/pkg/credentials/iam_aws.go +++ b/pkg/credentials/iam_aws.go @@ -22,6 +22,7 @@ import ( "encoding/json" "errors" "fmt" + "io/ioutil" "net" "net/http" "net/url" @@ -54,41 +55,10 @@ type IAM struct { const ( defaultIAMRoleEndpoint = "http://169.254.169.254" defaultECSRoleEndpoint = "http://169.254.170.2" + defaultSTSRoleEndpoint = "https://sts.amazonaws.com" defaultIAMSecurityCredsPath = "/latest/meta-data/iam/security-credentials/" ) -// https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html -func getEndpoint(endpoint string) (string, bool, error) { - ecsFullURI := os.Getenv("AWS_CONTAINER_CREDENTIALS_FULL_URI") - ecsURI := os.Getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") - - if endpoint != "" { - return endpoint, ecsURI != "" || ecsFullURI != "", nil - } - if ecsFullURI != "" { - u, err := url.Parse(ecsFullURI) - if err != nil { - return "", false, err - } - host := u.Hostname() - if host == "" { - return "", false, fmt.Errorf("can't parse host from uri: %s", ecsFullURI) - } - - if loopback, err := isLoopback(host); loopback { - return ecsFullURI, true, nil - } else if err != nil { - return "", false, err - } else { - return "", false, fmt.Errorf("host is not on a loopback address: %s", host) - } - } - if ecsURI != "" { - return fmt.Sprintf("%s%s", defaultECSRoleEndpoint, ecsURI), true, nil - } - return defaultIAMRoleEndpoint, false, nil -} - // NewIAM returns a pointer to a new Credentials object wrapping the IAM. func NewIAM(endpoint string) *Credentials { p := &IAM{ @@ -107,16 +77,61 @@ func (m *IAM) Retrieve() (Value, error) { var roleCreds ec2RoleCredRespBody var err error - endpoint, isEcsTask, err := getEndpoint(m.endpoint) - if err != nil { - return Value{}, err - } + endpoint := m.endpoint + switch { + case len(os.Getenv("AWS_WEB_IDENTITY_TOKEN_FILE")) > 0: + if len(endpoint) == 0 { + if len(os.Getenv("AWS_REGION")) > 0 { + endpoint = "sts." + os.Getenv("AWS_REGION") + ".amazonaws.com" + } else { + endpoint = defaultSTSRoleEndpoint + } + } + + creds := &STSWebIdentity{ + Client: m.Client, + stsEndpoint: endpoint, + roleARN: os.Getenv("AWS_ROLE_ARN"), + roleSessionName: os.Getenv("AWS_ROLE_SESSION_NAME"), + getWebIDTokenExpiry: func() (*WebIdentityToken, error) { + token, err := ioutil.ReadFile(os.Getenv("AWS_WEB_IDENTITY_TOKEN_FILE")) + if err != nil { + return nil, err + } + + return &WebIdentityToken{Token: string(token)}, nil + }, + } + + return creds.Retrieve() + + case len(os.Getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI")) > 0: + if len(endpoint) == 0 { + endpoint = fmt.Sprintf("%s%s", defaultECSRoleEndpoint, + os.Getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI")) + } + + roleCreds, err = getEcsTaskCredentials(m.Client, endpoint) + + case len(os.Getenv("AWS_CONTAINER_CREDENTIALS_FULL_URI")) > 0: + if len(endpoint) == 0 { + endpoint = os.Getenv("AWS_CONTAINER_CREDENTIALS_FULL_URI") + + var ok bool + if ok, err = isLoopback(endpoint); !ok { + if err == nil { + err = fmt.Errorf("uri host is not a loopback address: %s", endpoint) + } + break + } + } - if isEcsTask { roleCreds, err = getEcsTaskCredentials(m.Client, endpoint) - } else { + + default: roleCreds, err = getCredentials(m.Client, endpoint) } + if err != nil { return Value{}, err } @@ -276,8 +291,18 @@ func getCredentials(client *http.Client, endpoint string) (ec2RoleCredRespBody, return respCreds, nil } -// isLoopback identifies if a host is on a loopback address -func isLoopback(host string) (bool, error) { +// isLoopback identifies if a uri's host is on a loopback address +func isLoopback(uri string) (bool, error) { + u, err := url.Parse(uri) + if err != nil { + return false, err + } + + host := u.Hostname() + if len(host) == 0 { + return false, fmt.Errorf("can't parse host from uri: %s", uri) + } + ips, err := net.LookupHost(host) if err != nil { return false, err diff --git a/pkg/credentials/iam_aws_test.go b/pkg/credentials/iam_aws_test.go index d502a8ace..201514629 100644 --- a/pkg/credentials/iam_aws_test.go +++ b/pkg/credentials/iam_aws_test.go @@ -21,6 +21,7 @@ package credentials import ( "fmt" + "io/ioutil" "net/http" "net/http/httptest" "os" @@ -51,6 +52,27 @@ const credsRespEcsTaskTmpl = `{ "Expiration" : "%s" }` +const credsRespStsImpl = ` + + amzn1.account.AF6RHO7KZU5XRVQJGXK6HB56KR2A + client.5498841531868486423.1548@apps.example.com + + arn:aws:sts::123456789012:assumed-role/FederatedWebIdentityRole/app1 + AROACLKWSDQRAOEXAMPLE:app1 + + + token + secret + %s + accessKey + + www.amazon.com + + + ad4156e9-bce1-11e2-82e6-6b6efEXAMPLE + +` + func initTestFailServer() *httptest.Server { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "Not allowed", http.StatusBadRequest) @@ -91,6 +113,22 @@ func initEcsTaskTestServer(expireOn string) *httptest.Server { return server } +func initStsTestServer(expireOn string) *httptest.Server { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + required := []string{"RoleArn", "RoleSessionName", "WebIdentityToken", "Version"} + for _, field := range required { + if _, ok := r.URL.Query()[field]; !ok { + http.Error(w, fmt.Sprintf("%s missing", field), http.StatusBadRequest) + return + } + } + + fmt.Fprintf(w, credsRespStsImpl, expireOn) + })) + + return server +} + func TestIAMMalformedEndpoint(t *testing.T) { creds := NewIAM("%%%%") _, err := creds.Get() @@ -273,3 +311,44 @@ func TestEcsTaskFullURI(t *testing.T) { t.Error("Expected creds to be expired.") } } + +func TestSts(t *testing.T) { + server := initStsTestServer("2014-12-16T01:51:37Z") + defer server.Close() + p := &IAM{ + Client: http.DefaultClient, + endpoint: server.URL, + } + + f, err := ioutil.TempFile("", "minio-go") + if err != nil { + t.Errorf("Unexpected failure %s", err) + } + defer os.Remove(f.Name()) + f.Write([]byte("token")) + f.Close() + + os.Setenv("AWS_WEB_IDENTITY_TOKEN_FILE", f.Name()) + os.Setenv("AWS_ROLE_ARN", "arn:aws:sts::123456789012:assumed-role/FederatedWebIdentityRole/app1") + creds, err := p.Retrieve() + os.Unsetenv("AWS_WEB_IDENTITY_TOKEN_FILE") + os.Unsetenv("AWS_ROLE_ARN") + if err != nil { + t.Errorf("Unexpected failure %s", err) + } + if "accessKey" != creds.AccessKeyID { + t.Errorf("Expected \"accessKey\", got %s", creds.AccessKeyID) + } + + if "secret" != creds.SecretAccessKey { + t.Errorf("Expected \"secret\", got %s", creds.SecretAccessKey) + } + + if "token" != creds.SessionToken { + t.Errorf("Expected \"token\", got %s", creds.SessionToken) + } + + if !p.IsExpired() { + t.Error("Expected creds to be expired.") + } +} diff --git a/pkg/credentials/sts_web_identity.go b/pkg/credentials/sts_web_identity.go index 4d53bd236..216f48ac3 100644 --- a/pkg/credentials/sts_web_identity.go +++ b/pkg/credentials/sts_web_identity.go @@ -23,6 +23,7 @@ import ( "fmt" "net/http" "net/url" + "strconv" "time" ) @@ -75,6 +76,13 @@ type STSWebIdentity struct { // this token. // This is a customer provided function and is mandatory. getWebIDTokenExpiry func() (*WebIdentityToken, error) + + // roleARN is the Amazon Resource Name (ARN) of the role that the caller is + // assuming. + roleARN string + + // roleSessionName is the identifier for the assumed role session. + roleSessionName string } // NewSTSWebIdentity returns a pointer to a new @@ -95,7 +103,7 @@ func NewSTSWebIdentity(stsEndpoint string, getWebIDTokenExpiry func() (*WebIdent }), nil } -func getWebIdentityCredentials(clnt *http.Client, endpoint string, +func getWebIdentityCredentials(clnt *http.Client, endpoint, roleARN, roleSessionName string, getWebIDTokenExpiry func() (*WebIdentityToken, error)) (AssumeRoleWithWebIdentityResponse, error) { idToken, err := getWebIDTokenExpiry() if err != nil { @@ -104,8 +112,18 @@ func getWebIdentityCredentials(clnt *http.Client, endpoint string, v := url.Values{} v.Set("Action", "AssumeRoleWithWebIdentity") + if len(roleARN) > 0 { + v.Set("RoleArn", roleARN) + + if len(roleSessionName) == 0 { + roleSessionName = strconv.FormatInt(time.Now().UnixNano(), 10) + } + v.Set("RoleSessionName", roleSessionName) + } v.Set("WebIdentityToken", idToken.Token) - v.Set("DurationSeconds", fmt.Sprintf("%d", idToken.Expiry)) + if idToken.Expiry > 0 { + v.Set("DurationSeconds", fmt.Sprintf("%d", idToken.Expiry)) + } v.Set("Version", "2011-06-15") u, err := url.Parse(endpoint) @@ -141,7 +159,7 @@ func getWebIdentityCredentials(clnt *http.Client, endpoint string, // Retrieve retrieves credentials from the MinIO service. // Error will be returned if the request fails. func (m *STSWebIdentity) Retrieve() (Value, error) { - a, err := getWebIdentityCredentials(m.Client, m.stsEndpoint, m.getWebIDTokenExpiry) + a, err := getWebIdentityCredentials(m.Client, m.stsEndpoint, m.roleARN, m.roleSessionName, m.getWebIDTokenExpiry) if err != nil { return Value{}, err } From 5d89ff672a3f7aacb69e7baac225017c890e7d0e Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 10 Jan 2020 17:18:37 -0800 Subject: [PATCH 092/215] return notImplemented on listObjectsV2 call failure (#1212) --- api-list.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/api-list.go b/api-list.go index a82c75cf8..7bda405b3 100644 --- a/api-list.go +++ b/api-list.go @@ -19,7 +19,6 @@ package minio import ( "context" - "errors" "fmt" "net/http" "net/url" @@ -299,7 +298,10 @@ func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken s // This is an additional verification check to make // sure proper responses are received. if listBucketResult.IsTruncated && listBucketResult.NextContinuationToken == "" { - return listBucketResult, errors.New("Truncated response should have continuation token set") + return listBucketResult, ErrorResponse{ + Code: "NotImplemented", + Message: "Truncated response should have continuation token set", + } } for i, obj := range listBucketResult.Contents { From 71edb6cfe89c1944cc9574ddd95abf5409bb3bd8 Mon Sep 17 00:00:00 2001 From: poornas Date: Fri, 10 Jan 2020 19:15:53 -0800 Subject: [PATCH 093/215] Filter out X-Cache and X-Cache-Lookup from metadata (#1211) --- api-stat.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api-stat.go b/api-stat.go index 525253a3a..eb3b58510 100644 --- a/api-stat.go +++ b/api-stat.go @@ -91,6 +91,8 @@ func extractObjMetadata(header http.Header) http.Header { "Last-Modified", "Content-Type", "Expires", + "X-Cache", + "X-Cache-Lookup", }, defaultFilterKeys...) return filterHeader(header, filterKeys) } From 6d06ac0ffd73c23f82825e4c237b1a8e834356cc Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 10 Jan 2020 20:18:35 -0800 Subject: [PATCH 094/215] detect snowball server automatically using server header (#1213) --- api-error-response.go | 4 ++++ api-get-object.go | 18 ++++++++++++++++-- api-list.go | 12 ++++++++++++ bucket-cache.go | 4 +++- 4 files changed, 35 insertions(+), 3 deletions(-) diff --git a/api-error-response.go b/api-error-response.go index 726fdd274..4fda64856 100644 --- a/api-error-response.go +++ b/api-error-response.go @@ -51,6 +51,9 @@ type ErrorResponse struct { // only in HEAD bucket and ListObjects response. Region string + // Captures the server string returned in response header. + Server string + // Underlying HTTP status code for the returned error StatusCode int `xml:"-" json:"-"` } @@ -105,6 +108,7 @@ func httpRespToErrorResponse(resp *http.Response, bucketName, objectName string) errResp := ErrorResponse{ StatusCode: resp.StatusCode, + Server: resp.Header.Get("Server"), } err := xmlDecoder(resp.Body, &errResp) diff --git a/api-get-object.go b/api-get-object.go index 5654a602f..c73f4a6b4 100644 --- a/api-get-object.go +++ b/api-get-object.go @@ -45,6 +45,14 @@ func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName return nil, err } + // Detect if snowball is server location we are talking to. + var snowball bool + if location, ok := c.bucketLocCache.Get(bucketName); ok { + if location == "snowball" { + snowball = true + } + } + var httpReader io.ReadCloser var objectInfo ObjectInfo var err error @@ -136,7 +144,10 @@ func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName } else if req.settingObjectInfo { // Request is just to get objectInfo. // Remove range header if already set, for stat Operations to get original file size. delete(opts.headers, "Range") - if etag != "" { + // Check whether this is snowball + // if yes do not use If-Match feature + // it doesn't work. + if etag != "" && !snowball { opts.SetMatchETag(etag) } objectInfo, err := c.statObject(ctx, bucketName, objectName, StatObjectOptions{opts}) @@ -159,7 +170,10 @@ func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName // new ones when they haven't been already. // All readAt requests are new requests. if req.DidOffsetChange || !req.beenRead { - if etag != "" { + // Check whether this is snowball + // if yes do not use If-Match feature + // it doesn't work. + if etag != "" && !snowball { opts.SetMatchETag(etag) } if httpReader != nil { diff --git a/api-list.go b/api-list.go index 7bda405b3..c48201ce7 100644 --- a/api-list.go +++ b/api-list.go @@ -101,6 +101,12 @@ func (c Client) ListBucketsWithContext(ctx context.Context) ([]BucketInfo, error // func (c Client) ListObjectsV2WithMetadata(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { + // Check whether this is snowball region, if yes ListObjectsV2 doesn't work, fallback to listObjectsV1. + if location, ok := c.bucketLocCache.Get(bucketName); ok { + if location == "snowball" { + return c.ListObjects(bucketName, objectPrefix, recursive, doneCh) + } + } return c.listObjectsV2(bucketName, objectPrefix, recursive, true, doneCh) } @@ -210,6 +216,12 @@ func (c Client) listObjectsV2(bucketName, objectPrefix string, recursive, metada // } // func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { + // Check whether this is snowball region, if yes ListObjectsV2 doesn't work, fallback to listObjectsV1. + if location, ok := c.bucketLocCache.Get(bucketName); ok { + if location == "snowball" { + return c.ListObjects(bucketName, objectPrefix, recursive, doneCh) + } + } return c.listObjectsV2(bucketName, objectPrefix, recursive, false, doneCh) } diff --git a/bucket-cache.go b/bucket-cache.go index db8e8e6ce..5e46bf9a7 100644 --- a/bucket-cache.go +++ b/bucket-cache.go @@ -126,7 +126,9 @@ func processBucketLocationResponse(resp *http.Response, bucketName string) (buck // succeed if possible based on their policy. switch errResp.Code { case "NotImplemented": - fallthrough + if errResp.Server == "AmazonSnowball" { + return "snowball", nil + } case "AuthorizationHeaderMalformed": fallthrough case "InvalidRegion": From ae2a6fd9071da650a158332edb8c1d998de430d5 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Sat, 11 Jan 2020 11:00:19 +0530 Subject: [PATCH 095/215] Add ObjectTagging Support (#1206) Add new ObjectTagging APIs. Also, add ObjectTagging support in PutObject, CopyObject and other existing APIs. Changes are as per the documentation here https://docs.aws.amazon.com/AmazonS3/latest/dev/object-tagging.html This is a precursor to ObjectTagging support on MinIO server. Refer minio/minio#8446 Co-Authored-By: poornas --- api-compose-object.go | 100 ++++++++--- api-compose-object_test.go | 2 +- api-object-tagging.go | 163 ++++++++++++++++++ api-put-object-copy.go | 10 +- api-put-object.go | 4 + api-s3-datatypes.go | 16 ++ api-stat.go | 2 +- api.go | 2 +- constants.go | 5 + docs/API.md | 215 ++++++++++++++++++++++++ examples/s3/copyobject-with-new-tags.go | 81 +++++++++ examples/s3/getobjecttagging.go | 47 ++++++ examples/s3/putobject-with-tags.go | 61 +++++++ examples/s3/putobjecttagging.go | 49 ++++++ examples/s3/removeobjecttagging.go | 45 +++++ pkg/s3utils/utils.go | 25 +++ 16 files changed, 797 insertions(+), 30 deletions(-) create mode 100644 api-object-tagging.go create mode 100644 examples/s3/copyobject-with-new-tags.go create mode 100644 examples/s3/getobjecttagging.go create mode 100644 examples/s3/putobject-with-tags.go create mode 100644 examples/s3/putobjecttagging.go create mode 100644 examples/s3/removeobjecttagging.go diff --git a/api-compose-object.go b/api-compose-object.go index 748b558ce..c9e44c556 100644 --- a/api-compose-object.go +++ b/api-compose-object.go @@ -36,18 +36,53 @@ import ( // created via server-side copy requests, using the Compose API. type DestinationInfo struct { bucket, object string - encryption encrypt.ServerSide + opts DestInfoOptions +} +// DestInfoOptions represents options specified by user for NewDestinationInfo call +type DestInfoOptions struct { + // `Encryption` is the key info for server-side-encryption with customer + // provided key. If it is nil, no encryption is performed. + Encryption encrypt.ServerSide + + // `userMeta` is the user-metadata key-value pairs to be set on the + // destination. The keys are automatically prefixed with `x-amz-meta-` + // if needed. If nil is passed, and if only a single source (of any + // size) is provided in the ComposeObject call, then metadata from the + // source is copied to the destination. // if no user-metadata is provided, it is copied from source // (when there is only once source object in the compose // request) - userMetadata map[string]string + UserMeta map[string]string + + // `userTags` is the user defined object tags to be set on destination. + // This will be set only if the `replaceTags` field is set to true. + // Otherwise this field is ignored + UserTags map[string]string + ReplaceTags bool +} + +// Process custom-metadata to remove a `x-amz-meta-` prefix if +// present and validate that keys are distinct (after this +// prefix removal). +func filterCustomMeta(userMeta map[string]string) (map[string]string, error) { + m := make(map[string]string) + for k, v := range userMeta { + if strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") { + k = k[len("x-amz-meta-"):] + } + if _, ok := m[k]; ok { + return nil, ErrInvalidArgument(fmt.Sprintf("Cannot add both %s and x-amz-meta-%s keys as custom metadata", k, k)) + } + m[k] = v + } + return m, nil } // NewDestinationInfo - creates a compose-object/copy-source // destination info object. // -// `encSSEC` is the key info for server-side-encryption with customer +// `sse` is the key info for server-side-encryption with customer // provided key. If it is nil, no encryption is performed. // // `userMeta` is the user-metadata key-value pairs to be set on the @@ -63,26 +98,41 @@ func NewDestinationInfo(bucket, object string, sse encrypt.ServerSide, userMeta if err = s3utils.CheckValidObjectName(object); err != nil { return d, err } - - // Process custom-metadata to remove a `x-amz-meta-` prefix if - // present and validate that keys are distinct (after this - // prefix removal). - m := make(map[string]string) - for k, v := range userMeta { - if strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") { - k = k[len("x-amz-meta-"):] - } - if _, ok := m[k]; ok { - return d, ErrInvalidArgument(fmt.Sprintf("Cannot add both %s and x-amz-meta-%s keys as custom metadata", k, k)) - } - m[k] = v + m, err := filterCustomMeta(userMeta) + if err != nil { + return d, err } + opts := DestInfoOptions{ + Encryption: sse, + UserMeta: m, + UserTags: nil, + ReplaceTags: false, + } + return DestinationInfo{ + bucket: bucket, + object: object, + opts: opts, + }, nil +} +// NewDestinationInfoWithOptions - creates a compose-object/copy-source +// destination info object. +func NewDestinationInfoWithOptions(bucket, object string, destOpts DestInfoOptions) (d DestinationInfo, err error) { + // Input validation. + if err = s3utils.CheckValidBucketName(bucket); err != nil { + return d, err + } + if err = s3utils.CheckValidObjectName(object); err != nil { + return d, err + } + destOpts.UserMeta, err = filterCustomMeta(destOpts.UserMeta) + if err != nil { + return d, err + } return DestinationInfo{ - bucket: bucket, - object: object, - encryption: sse, - userMetadata: m, + bucket: bucket, + object: object, + opts: destOpts, }, nil } @@ -93,14 +143,14 @@ func NewDestinationInfo(bucket, object string, sse encrypt.ServerSide, userMeta // `REPLACE`, so that metadata headers from the source are not copied // over. func (d *DestinationInfo) getUserMetaHeadersMap(withCopyDirectiveHeader bool) map[string]string { - if len(d.userMetadata) == 0 { + if len(d.opts.UserMeta) == 0 { return nil } r := make(map[string]string) if withCopyDirectiveHeader { r["x-amz-metadata-directive"] = "REPLACE" } - for k, v := range d.userMetadata { + for k, v := range d.opts.UserMeta { if isAmzHeader(k) || isStandardHeader(k) || isStorageClassHeader(k) { r[k] = v } else { @@ -447,7 +497,7 @@ func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo metaHeaders[k] = v } - uploadID, err := c.newUploadID(ctx, dst.bucket, dst.object, PutObjectOptions{ServerSideEncryption: dst.encryption, UserMetadata: metaHeaders}) + uploadID, err := c.newUploadID(ctx, dst.bucket, dst.object, PutObjectOptions{ServerSideEncryption: dst.opts.Encryption, UserMetadata: metaHeaders}) if err != nil { return err } @@ -461,8 +511,8 @@ func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo encrypt.SSECopy(src.encryption).Marshal(h) } // Add destination encryption headers - if dst.encryption != nil { - dst.encryption.Marshal(h) + if dst.opts.Encryption != nil { + dst.opts.Encryption.Marshal(h) } // calculate start/end indices of parts after diff --git a/api-compose-object_test.go b/api-compose-object_test.go index e9f04d3a6..ddb26a114 100644 --- a/api-compose-object_test.go +++ b/api-compose-object_test.go @@ -130,7 +130,7 @@ func TestGetUserMetaHeadersMap(t *testing.T) { "x-amz-grant-write": "test@exo.ch", } - destInfo := &DestinationInfo{"bucket", "object", nil, userMetadata} + destInfo, _ := NewDestinationInfo("bucket", "object", nil, userMetadata) r := destInfo.getUserMetaHeadersMap(true) diff --git a/api-object-tagging.go b/api-object-tagging.go new file mode 100644 index 000000000..fdf8fab91 --- /dev/null +++ b/api-object-tagging.go @@ -0,0 +1,163 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "context" + "encoding/xml" + "io/ioutil" + "net/http" + "net/url" + + "github.com/minio/minio-go/v6/pkg/s3utils" +) + +// PutObjectTagging replaces or creates object tag(s) +func (c Client) PutObjectTagging(bucketName, objectName string, objectTags map[string]string) error { + return c.PutObjectTaggingWithContext(context.Background(), bucketName, objectName, objectTags) +} + +// PutObjectTaggingWithContext replaces or creates object tag(s) with a context to control cancellations +// and timeouts. +func (c Client) PutObjectTaggingWithContext(ctx context.Context, bucketName, objectName string, objectTags map[string]string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("tagging", "") + + tags := make([]tag, 0) + for k, v := range objectTags { + t := tag{ + Key: k, + Value: v, + } + tags = append(tags, t) + } + + // Prepare Tagging struct + reqBody := tagging{ + TagSet: tagSet{ + Tags: tags, + }, + } + + reqBytes, err := xml.Marshal(reqBody) + if err != nil { + return err + } + + reqMetadata := requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: urlValues, + contentBody: bytes.NewReader(reqBytes), + contentLength: int64(len(reqBytes)), + contentMD5Base64: sumMD5Base64(reqBytes), + } + + // Execute PUT to set a object tagging. + resp, err := c.executeMethod(ctx, "PUT", reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, bucketName, objectName) + } + } + return nil +} + +// GetObjectTagging fetches object tag(s) +func (c Client) GetObjectTagging(bucketName, objectName string) (string, error) { + return c.GetObjectTaggingWithContext(context.Background(), bucketName, objectName) +} + +// GetObjectTaggingWithContext fetches object tag(s) with a context to control cancellations +// and timeouts. +func (c Client) GetObjectTaggingWithContext(ctx context.Context, bucketName, objectName string) (string, error) { + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("tagging", "") + + // Execute GET on object to get object tag(s) + resp, err := c.executeMethod(ctx, "GET", requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: urlValues, + }) + + defer closeResponse(resp) + if err != nil { + return "", err + } + + if resp != nil { + if resp.StatusCode != http.StatusOK { + return "", httpRespToErrorResponse(resp, bucketName, objectName) + } + } + + tagBuf, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + + return string(tagBuf), err +} + +// RemoveObjectTagging deletes object tag(s) +func (c Client) RemoveObjectTagging(bucketName, objectName string) error { + return c.RemoveObjectTaggingWithContext(context.Background(), bucketName, objectName) +} + +// RemoveObjectTaggingWithContext removes object tag(s) with a context to control cancellations +// and timeouts. +func (c Client) RemoveObjectTaggingWithContext(ctx context.Context, bucketName, objectName string) error { + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("tagging", "") + + // Execute DELETE on object to remove object tag(s) + resp, err := c.executeMethod(ctx, "DELETE", requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: urlValues, + }) + + defer closeResponse(resp) + if err != nil { + return err + } + + if resp != nil { + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, bucketName, objectName) + } + } + return err +} diff --git a/api-put-object-copy.go b/api-put-object-copy.go index 19e58add8..ce48674b0 100644 --- a/api-put-object-copy.go +++ b/api-put-object-copy.go @@ -24,6 +24,7 @@ import ( "net/http" "github.com/minio/minio-go/v6/pkg/encrypt" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // CopyObject - copy a source object into a new object @@ -39,6 +40,11 @@ func (c Client) CopyObjectWithProgress(dst DestinationInfo, src SourceInfo, prog header[k] = v } + if dst.opts.ReplaceTags && len(dst.opts.UserTags) != 0 { + header.Set(amzTaggingHeaderDirective, "REPLACE") + header.Set(amzTaggingHeader, s3utils.TagEncode(dst.opts.UserTags)) + } + var err error var size int64 // If progress bar is specified, size should be requested as well initiate a StatObject request. @@ -53,8 +59,8 @@ func (c Client) CopyObjectWithProgress(dst DestinationInfo, src SourceInfo, prog encrypt.SSECopy(src.encryption).Marshal(header) } - if dst.encryption != nil { - dst.encryption.Marshal(header) + if dst.opts.Encryption != nil { + dst.opts.Encryption.Marshal(header) } for k, v := range dst.getUserMetaHeadersMap(true) { header.Set(k, v) diff --git a/api-put-object.go b/api-put-object.go index bc0848bbf..8b327ebf0 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -35,6 +35,7 @@ import ( // PutObjectOptions represents options specified by user for PutObject call type PutObjectOptions struct { UserMetadata map[string]string + UserTags map[string]string Progress io.Reader ContentType string ContentEncoding string @@ -100,6 +101,9 @@ func (opts PutObjectOptions) Header() (header http.Header) { if opts.WebsiteRedirectLocation != "" { header[amzWebsiteRedirectLocation] = []string{opts.WebsiteRedirectLocation} } + if len(opts.UserTags) != 0 { + header[amzTaggingHeader] = []string{s3utils.TagEncode(opts.UserTags)} + } for k, v := range opts.UserMetadata { if !isAmzHeader(k) && !isStandardHeader(k) && !isStorageClassHeader(k) { header["X-Amz-Meta-"+k] = []string{v} diff --git a/api-s3-datatypes.go b/api-s3-datatypes.go index a6b125522..deb56e39d 100644 --- a/api-s3-datatypes.go +++ b/api-s3-datatypes.go @@ -243,3 +243,19 @@ type deleteMultiObjectsResult struct { DeletedObjects []deletedObject `xml:"Deleted"` UnDeletedObjects []nonDeletedObject `xml:"Error"` } + +type tagging struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Tagging"` + TagSet tagSet +} + +type tagSet struct { + XMLName xml.Name `xml:"TagSet"` + Tags []tag +} + +type tag struct { + XMLName xml.Name `xml:"Tag"` + Key string `xml:"Key"` + Value string `xml:"Value"` +} diff --git a/api-stat.go b/api-stat.go index eb3b58510..45844c9cb 100644 --- a/api-stat.go +++ b/api-stat.go @@ -78,7 +78,7 @@ var defaultFilterKeys = []string{ "x-amz-id-2", "Content-Security-Policy", "X-Xss-Protection", - + amzTaggingHeaderCount, // Add new headers to be ignored. } diff --git a/api.go b/api.go index 9b6b260d0..235407290 100644 --- a/api.go +++ b/api.go @@ -824,7 +824,7 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R case signerType.IsV2(): // Add signature version '2' authorization header. req = s3signer.SignV2(*req, accessKeyID, secretAccessKey, isVirtualHost) - case metadata.objectName != "" && method == "PUT" && metadata.customHeader.Get("X-Amz-Copy-Source") == "" && !c.secure: + case metadata.objectName != "" && metadata.queryValues == nil && method == "PUT" && metadata.customHeader.Get("X-Amz-Copy-Source") == "" && !c.secure: // Streaming signature is used by default for a PUT object request. Additionally we also // look if the initialized client is secure, if yes then we don't need to perform // streaming signature. diff --git a/constants.go b/constants.go index ac472a631..7f907c8b2 100644 --- a/constants.go +++ b/constants.go @@ -60,3 +60,8 @@ const amzStorageClass = "X-Amz-Storage-Class" // Website redirect location header constant const amzWebsiteRedirectLocation = "X-Amz-Website-Redirect-Location" + +// Tagging header constants +const amzTaggingHeader = "X-Amz-Tagging" +const amzTaggingHeaderDirective = "X-Amz-Tagging-Directive" +const amzTaggingHeaderCount = "X-Amz-Tagging-Count" diff --git a/docs/API.md b/docs/API.md index f17487de9..dbd9ecfee 100644 --- a/docs/API.md +++ b/docs/API.md @@ -73,6 +73,13 @@ func main() { | | [`PutObjectRetention`](#PutObjectRetention) | | | | | | [`GetObjectRetention`](#GetObjectRetention) | | | | | | [`SelectObjectContent`](#SelectObjectContent) | | +| | [`PutObjectTagging`](#PutObjectTagging) | | +| | [`PutObjectTaggingWithContext`](#PutObjectTaggingWithContext) | | +| | [`GetObjectTagging`](#GetObjectTagging) | | +| | [`GetObjectTaggingWithContext`](#GetObjectTaggingWithContext) | | +| | [`RemoveObjectTagging`](#RemoveObjectTagging) | | +| | [`RemoveObjectTaggingWithContext`](#RemoveObjectTaggingWithContext) | | + ## 1. Constructor @@ -582,6 +589,7 @@ __minio.PutObjectOptions__ |Field | Type | Description | |:--- |:--- | :--- | | `opts.UserMetadata` | _map[string]string_ | Map of user metadata| +| `opts.UserTags` | _map[string]string_ | Map of user object tags | | `opts.Progress` | _io.Reader_ | Reader to fetch progress of an upload | | `opts.ContentType` | _string_ | Content type of object, e.g "application/text" | | `opts.ContentEncoding` | _string_ | Content encoding of object, e.g "gzip" | @@ -903,6 +911,68 @@ if err != nil { } ``` + +### NewDestinationInfoWithOptions(bucket, object string, destOpts DestInfoOptions) (DestinationInfo, error) +Construct a `DestinationInfo` object that can be used as the destination object for server-side copying operations like `CopyObject` and `ComposeObject`. + +__Parameters__ + +| Param | Type | Description | +| :--- | :--- | :--- | +| `bucket` | _string_ | Name of the destination bucket | +| `object` | _string_ | Name of the destination object | +| `destOpts` | _minio.DestInfoOptions_ | Pointer to struct that allows user to set optional custom metadata, user tags, and server side encryption parameters. | + +__Example__ + +```go +// No encryption parameter. +src := minio.NewSourceInfo("bucket", "object", nil) +tags := map[string]string{ + "Tag1": "Value1", + "Tag2": "Value2", +} +dst, err := minio.NewDestinationInfoWithOptions("bucket", "object", minio.DestInfoOptions{ + UserTags: tags, ReplaceTags: true, +}) +if err != nil { + fmt.Println(err) + return +} + +// Copy object call +err = minioClient.CopyObject(dst, src) +if err != nil { + fmt.Println(err) + return +} +``` + +```go +src := minio.NewSourceInfo("bucket", "object", nil) + +// With encryption parameter. +sseDst := encrypt.DefaultPBKDF([]byte("password"), []byte("salt")) +tags := map[string]string{ + "Tag1": "Value1", + "Tag2": "Value2", +} +dst, err := minio.NewDestinationInfoWithOptions("bucket", "object", minio.DestInfoOptions{ + Encryption: sseDst, UserTags: tags, ReplaceTags: true, +}) +if err != nil { + fmt.Println(err) + return +} + +// Copy object call +err = minioClient.CopyObject(dst, src) +if err != nil { + fmt.Println(err) + return +} +``` + ### FPutObject(bucketName, objectName, filePath, opts PutObjectOptions) (length int64, err error) Uploads contents from a file to objectName. @@ -1249,6 +1319,151 @@ __Return Values__ } ``` + +### PutObjectTagging(bucketName, objectName string, objectTags map[string]string) error +Adds or replace Object Tags to the given object + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`objectTags` | _map[string]string_ | Map with Object Tag's Key and Value | + +__Example__ + + +```go +err = minioClient.PutObjectTagging(bucketName, objectName, objectTags) +if err != nil { + fmt.Println(err) + return +} +``` + + +### PutObjectTaggingWithContext(ctx context.Context, sssbucketName, objectName string, objectTags map[string]string) error +Identical to PutObjectTagging, but allows setting context to allow controlling context cancellations and timeouts. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`ctx` | _context.Context_ |Request context | +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`objectTags` | _map[string]string_ | Map with Object Tag's Key and Value | + +__Example__ + + +```go +err = minioClient.PutObjectTaggingWithContext(ctx, bucketName, objectName, objectTags) +if err != nil { + fmt.Println(err) + return +} +``` + + +### GetObjectTagging(bucketName, objectName string) (string, error) +Fetch Object Tags from the given object + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | + +__Example__ + + +```go +tags, err = minioClient.GetObjectTagging(bucketName, objectName) +if err != nil { + fmt.Println(err) + return +} +fmt.Printf("Fetched Tags: %s", tags) +``` + + +### GetObjectTaggingWithContext(ctx context.Context, bucketName, objectName string) (string, error) +Identical to GetObjectTagging, but allows setting context to allow controlling context cancellations and timeouts. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`ctx` | _context.Context_ |Request context | +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | + +__Example__ + + +```go +tags, err = minioClient.GetObjectTaggingWithContext(ctx, bucketName, objectName) +if err != nil { + fmt.Println(err) + return +} +fmt.Printf("Fetched Tags: %s", tags) +``` + + +### RemoveObjectTagging(bucketName, objectName string) error +Remove Object Tags from the given object + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | + +__Example__ + + +```go +err = minioClient.RemoveObjectTagging(bucketName, objectName) +if err != nil { + fmt.Println(err) + return +} +``` + + +### RemoveObjectTaggingWithContext(ctx context.Context, bucketName, objectName string) error +Identical to RemoveObjectTagging, but allows setting context to allow controlling context cancellations and timeouts. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`ctx` | _context.Context_ |Request context | +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | + +__Example__ + + +```go +err = minioClient.RemoveObjectTaggingWithContext(ctx, bucketName, objectName) +if err != nil { + fmt.Println(err) + return +} +``` + ### RemoveIncompleteUpload(bucketName, objectName string) error Removes a partially uploaded object. diff --git a/examples/s3/copyobject-with-new-tags.go b/examples/s3/copyobject-with-new-tags.go new file mode 100644 index 000000000..bf9a12d39 --- /dev/null +++ b/examples/s3/copyobject-with-new-tags.go @@ -0,0 +1,81 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + "time" + + "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and + // my-objectname are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // Enable trace. + // s3Client.TraceOn(os.Stderr) + + // Source object + src := minio.NewSourceInfo("my-sourcebucketname", "my-sourceobjectname", nil) + + // All following conditions are allowed and can be combined together. + + // Set modified condition, copy object modified since 2014 April. + src.SetModifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) + + // Set unmodified condition, copy object unmodified since 2014 April. + // src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) + + // Set matching ETag condition, copy object which matches the following ETag. + // src.SetMatchETagCond("31624deb84149d2f8ef9c385918b653a") + + // Set matching ETag except condition, copy object which does not match the following ETag. + // src.SetMatchETagExceptCond("31624deb84149d2f8ef9c385918b653a") + + tags := map[string]string{ + "Tag1": "Value1", + "Tag2": "Value2", + } + + // Destination object + dst, err := minio.NewDestinationInfoWithOptions("my-bucketname", "my-objectname", minio.DestInfoOptions{ + UserTags: tags, ReplaceTags: true, + }) + if err != nil { + log.Fatalln(err) + } + // Initiate copy object. + err = s3Client.CopyObject(dst, src) + if err != nil { + log.Fatalln(err) + } + log.Println("Copied source object /my-sourcebucketname/my-sourceobjectname to destination /my-bucketname/my-objectname Successfully.") +} diff --git a/examples/s3/getobjecttagging.go b/examples/s3/getobjecttagging.go new file mode 100644 index 000000000..fb4b38396 --- /dev/null +++ b/examples/s3/getobjecttagging.go @@ -0,0 +1,47 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + "log" + + "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and + // my-objectname are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + tags, err := s3Client.GetObjectTagging("my-bucketname", "my-objectname") + if err != nil { + log.Fatalln(err) + } + fmt.Printf("Fetched Object Tags: %s", tags) +} diff --git a/examples/s3/putobject-with-tags.go b/examples/s3/putobject-with-tags.go new file mode 100644 index 000000000..5c7ae6bea --- /dev/null +++ b/examples/s3/putobject-with-tags.go @@ -0,0 +1,61 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + "os" + + "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and + // my-objectname are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + object, err := os.Open("my-testfile") + if err != nil { + log.Fatalln(err) + } + defer object.Close() + objectStat, err := object.Stat() + if err != nil { + log.Fatalln(err) + } + tags := map[string]string{ + "Tag1": "Value1", + "Tag2": "Value2", + } + n, err := s3Client.PutObject("my-bucketname", "my-objectname", object, objectStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream", UserTags: tags}) + if err != nil { + log.Fatalln(err) + } + log.Println("Uploaded", "my-objectname", " of size: ", n, "Successfully.") +} diff --git a/examples/s3/putobjecttagging.go b/examples/s3/putobjecttagging.go new file mode 100644 index 000000000..a7532e694 --- /dev/null +++ b/examples/s3/putobjecttagging.go @@ -0,0 +1,49 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and + // my-objectname are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + tags := map[string]string{ + "Tag1": "Value1", + "Tag2": "Value2", + } + err = s3Client.PutObjectTagging("my-bucketname", "my-objectname", tags) + if err != nil { + log.Fatalln(err) + } +} diff --git a/examples/s3/removeobjecttagging.go b/examples/s3/removeobjecttagging.go new file mode 100644 index 000000000..3fbae638a --- /dev/null +++ b/examples/s3/removeobjecttagging.go @@ -0,0 +1,45 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and + // my-objectname are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + err = s3Client.RemoveObjectTagging("my-bucketname", "my-objectname") + if err != nil { + log.Fatalln(err) + } +} diff --git a/pkg/s3utils/utils.go b/pkg/s3utils/utils.go index 9af2997b7..f1f6eea60 100644 --- a/pkg/s3utils/utils.go +++ b/pkg/s3utils/utils.go @@ -219,6 +219,31 @@ func QueryEncode(v url.Values) string { return buf.String() } +// TagEncode - encodes tag values in their URL encoded form. In +// addition to the percent encoding performed by urlEncodePath() used +// here, it also percent encodes '/' (forward slash) +func TagEncode(tags map[string]string) string { + if tags == nil { + return "" + } + var buf bytes.Buffer + keys := make([]string, 0, len(tags)) + for k := range tags { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + v := tags[k] + prefix := percentEncodeSlash(EncodePath(k)) + "=" + if buf.Len() > 0 { + buf.WriteByte('&') + } + buf.WriteString(prefix) + buf.WriteString(percentEncodeSlash(EncodePath(v))) + } + return buf.String() +} + // if object matches reserved string, no need to encode them var reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$") From db6ff93bcdfd818d61b76940458ec9d07e212d57 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 16 Jan 2020 13:30:22 -0800 Subject: [PATCH 096/215] fix: allow configurable part size with -1 content-length (#1215) Fixes #1214 Co-Authored-By: Krishnan Parthasarathi --- api-put-object-common.go | 16 +++++++++++++--- api_unit_test.go | 19 ++++++++++++++----- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/api-put-object-common.go b/api-put-object-common.go index a786d2a8e..78e0a2fdc 100644 --- a/api-put-object-common.go +++ b/api-put-object-common.go @@ -69,7 +69,9 @@ func isReadAt(reader io.Reader) (ok bool) { // func optimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCount int, partSize int64, lastPartSize int64, err error) { // object size is '-1' set it to 5TiB. + var unknownSize bool if objectSize == -1 { + unknownSize = true objectSize = maxMultipartPutObjectSize } @@ -86,9 +88,11 @@ func optimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCou return } - if objectSize > (int64(configuredPartSize) * maxPartsCount) { - err = ErrInvalidArgument("Part size * max_parts(10000) is lesser than input objectSize.") - return + if !unknownSize { + if objectSize > (int64(configuredPartSize) * maxPartsCount) { + err = ErrInvalidArgument("Part size * max_parts(10000) is lesser than input objectSize.") + return + } } if configuredPartSize < absMinPartSize { @@ -100,7 +104,13 @@ func optimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCou err = ErrInvalidArgument("Input part size is bigger than allowed maximum of 5GiB.") return } + partSizeFlt = float64(configuredPartSize) + if unknownSize { + // If input has unknown size and part size is configured + // keep it to maximum allowed as per 10000 parts. + objectSize = int64(configuredPartSize) * maxPartsCount + } } else { configuredPartSize = minPartSize // Use floats for part size for all calculations to avoid diff --git a/api_unit_test.go b/api_unit_test.go index fbfccdbb8..2dca213d4 100644 --- a/api_unit_test.go +++ b/api_unit_test.go @@ -136,11 +136,6 @@ func TestPartSize(t *testing.T) { if partSize != minPartSize { t.Fatalf("Error: expecting part size of %v: got %v instead", minPartSize, partSize) } - // if stream and client configured min part size - _, _, _, err = optimalPartInfo(-1, minPartSize) - if err == nil { - t.Fatal("Error:", err) - } // if stream and using default optimal part size determined by sdk totalPartsCount, partSize, lastPartSize, err = optimalPartInfo(-1, 0) if err != nil { @@ -155,6 +150,20 @@ func TestPartSize(t *testing.T) { if lastPartSize != 671088640 { t.Fatalf("Error: expecting last part size of 671088640: got %v instead", lastPartSize) } + + totalPartsCount, partSize, lastPartSize, err = optimalPartInfo(-1, 64*1024*1024) + if err != nil { + t.Fatal("Error:", err) + } + if totalPartsCount != 10000 { + t.Fatalf("Error: expecting total parts count of 10000: got %v instead", totalPartsCount) + } + if partSize != 67108864 { + t.Fatalf("Error: expecting part size of 67108864: got %v instead", partSize) + } + if lastPartSize != 67108864 { + t.Fatalf("Error: expecting part size of 67108864: got %v instead", lastPartSize) + } } // TestMakeTargetURL - testing makeTargetURL() From 66cf57d21ba4b64cdb614f21a468a04bdf21e74d Mon Sep 17 00:00:00 2001 From: Siarhei Navatski Date: Fri, 17 Jan 2020 17:09:06 +0300 Subject: [PATCH 097/215] Remove empty X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id request header (#1216) --- pkg/encrypt/server-side.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/encrypt/server-side.go b/pkg/encrypt/server-side.go index ac0b69a02..cabc65067 100644 --- a/pkg/encrypt/server-side.go +++ b/pkg/encrypt/server-side.go @@ -188,7 +188,9 @@ func (s kms) Type() Type { return KMS } func (s kms) Marshal(h http.Header) { h.Set(sseGenericHeader, "aws:kms") - h.Set(sseKmsKeyID, s.key) + if s.key != "" { + h.Set(sseKmsKeyID, s.key) + } if s.hasContext { h.Set(sseEncryptionContext, base64.StdEncoding.EncodeToString(s.context)) } From c215d02cf7b366b99ad278ba06692407424d366c Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 20 Jan 2020 08:47:37 -0800 Subject: [PATCH 098/215] fix: let net/http handle rootCAs properly by default (#1218) Fixes #1191 --- transport.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/transport.go b/transport.go index 34efa8980..89e4297b9 100644 --- a/transport.go +++ b/transport.go @@ -21,7 +21,6 @@ package minio import ( "crypto/tls" - "crypto/x509" "net" "net/http" "time" @@ -54,17 +53,8 @@ var DefaultTransport = func(secure bool) (http.RoundTripper, error) { } if secure { - rootCAs, _ := x509.SystemCertPool() - if rootCAs == nil { - // In some systems (like Windows) system cert pool is - // not supported or no certificates are present on the - // system - so we create a new cert pool. - rootCAs = x509.NewCertPool() - } - // Keep TLS config. tlsConfig := &tls.Config{ - RootCAs: rootCAs, // Can't use SSLv3 because of POODLE and BEAST // Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher // Can't use TLSv1.1 because of RC4 cipher usage From 9d882c9f270a17c1d5cfce682fee79bf87f9bd81 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 22 Jan 2020 14:40:49 -0800 Subject: [PATCH 099/215] fix: avoid shadowing inside go routine (#1219) --- api-put-object-multipart.go | 7 +++---- api-put-object-streaming.go | 32 +++++++++++--------------------- api-put-object.go | 18 +++++++++--------- transport.go | 12 +----------- 4 files changed, 24 insertions(+), 45 deletions(-) diff --git a/api-put-object-multipart.go b/api-put-object-multipart.go index 3b39ed2bd..864acc8bc 100644 --- a/api-put-object-multipart.go +++ b/api-put-object-multipart.go @@ -137,11 +137,10 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje } // Proceed to upload the part. - var objPart ObjectPart - objPart, err = c.uploadPart(ctx, bucketName, objectName, uploadID, rd, partNumber, + objPart, uerr := c.uploadPart(ctx, bucketName, objectName, uploadID, rd, partNumber, md5Base64, sha256Hex, int64(length), opts.ServerSideEncryption) - if err != nil { - return totalUploadedSize, err + if uerr != nil { + return totalUploadedSize, uerr } // Save successfully uploaded part metadata. diff --git a/api-put-object-streaming.go b/api-put-object-streaming.go index 46b93e602..5add2afcf 100644 --- a/api-put-object-streaming.go +++ b/api-put-object-streaming.go @@ -67,12 +67,12 @@ type uploadedPartRes struct { Error error // Any error encountered while uploading the part. PartNum int // Number of the part uploaded. Size int64 // Size of the part uploaded. - Part *ObjectPart + Part ObjectPart } type uploadPartReq struct { - PartNum int // Number of the part uploaded. - Part *ObjectPart // Size of the part uploaded. + PartNum int // Number of the part uploaded. + Part ObjectPart // Size of the part uploaded. } // putObjectMultipartFromReadAt - Uploads files bigger than 128MiB. @@ -139,7 +139,7 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa // Send each part number to the channel to be processed. for p := 1; p <= totalPartsCount; p++ { - uploadPartsCh <- uploadPartReq{PartNum: p, Part: nil} + uploadPartsCh <- uploadPartReq{PartNum: p} } close(uploadPartsCh) // Receive each part number from the channel allowing three parallel uploads. @@ -164,13 +164,11 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa sectionReader := newHook(io.NewSectionReader(reader, readOffset, partSize), opts.Progress) // Proceed to upload the part. - var objPart ObjectPart - objPart, err = c.uploadPart(ctx, bucketName, objectName, uploadID, + objPart, err := c.uploadPart(ctx, bucketName, objectName, uploadID, sectionReader, uploadReq.PartNum, "", "", partSize, opts.ServerSideEncryption) if err != nil { uploadedPartsCh <- uploadedPartRes{ - Size: 0, Error: err, } // Exit the goroutine. @@ -178,14 +176,13 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa } // Save successfully uploaded part metadata. - uploadReq.Part = &objPart + uploadReq.Part = objPart // Send successful part info through the channel. uploadedPartsCh <- uploadedPartRes{ Size: objPart.Size, PartNum: uploadReq.PartNum, Part: uploadReq.Part, - Error: nil, } } }(partSize) @@ -198,18 +195,12 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa if uploadRes.Error != nil { return totalUploadedSize, uploadRes.Error } - // Retrieve each uploaded part and store it to be completed. - // part, ok := partsInfo[uploadRes.PartNum] - part := uploadRes.Part - if part == nil { - return 0, ErrInvalidArgument(fmt.Sprintf("Missing part number %d", uploadRes.PartNum)) - } // Update the totalUploadedSize. totalUploadedSize += uploadRes.Size // Store the parts to be completed in order. complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ - ETag: part.ETag, - PartNumber: part.PartNumber, + ETag: uploadRes.Part.ETag, + PartNumber: uploadRes.Part.PartNumber, }) } @@ -277,12 +268,11 @@ func (c Client) putObjectMultipartStreamNoChecksum(ctx context.Context, bucketNa if partNumber == totalPartsCount { partSize = lastPartSize } - var objPart ObjectPart - objPart, err = c.uploadPart(ctx, bucketName, objectName, uploadID, + objPart, uerr := c.uploadPart(ctx, bucketName, objectName, uploadID, io.LimitReader(hookReader, partSize), partNumber, "", "", partSize, opts.ServerSideEncryption) - if err != nil { - return totalUploadedSize, err + if uerr != nil { + return totalUploadedSize, uerr } // Save successfully uploaded part metadata. diff --git a/api-put-object.go b/api-put-object.go index 8b327ebf0..c760056cb 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -235,23 +235,23 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName defer debug.FreeOSMemory() for partNumber <= totalPartsCount { - length, rErr := io.ReadFull(reader, buf) - if rErr == io.EOF && partNumber > 1 { + length, rerr := io.ReadFull(reader, buf) + if rerr == io.EOF && partNumber > 1 { break } - if rErr != nil && rErr != io.ErrUnexpectedEOF && rErr != io.EOF { - return 0, rErr + if rerr != nil && rerr != io.ErrUnexpectedEOF && rerr != io.EOF { + return 0, rerr } + // Update progress reader appropriately to the latest offset // as we read from the source. rd := newHook(bytes.NewReader(buf[:length]), opts.Progress) // Proceed to upload the part. - var objPart ObjectPart - objPart, err = c.uploadPart(ctx, bucketName, objectName, uploadID, rd, partNumber, + objPart, uerr := c.uploadPart(ctx, bucketName, objectName, uploadID, rd, partNumber, "", "", int64(length), opts.ServerSideEncryption) - if err != nil { - return totalUploadedSize, err + if uerr != nil { + return totalUploadedSize, uerr } // Save successfully uploaded part metadata. @@ -265,7 +265,7 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName // For unknown size, Read EOF we break away. // We do not have to upload till totalPartsCount. - if rErr == io.EOF { + if rerr == io.EOF { break } } diff --git a/transport.go b/transport.go index 89e4297b9..00ee4dad6 100644 --- a/transport.go +++ b/transport.go @@ -24,8 +24,6 @@ import ( "net" "net/http" "time" - - "golang.org/x/net/http2" ) // DefaultTransport - this default transport is similar to @@ -53,20 +51,12 @@ var DefaultTransport = func(secure bool) (http.RoundTripper, error) { } if secure { - // Keep TLS config. - tlsConfig := &tls.Config{ + tr.TLSClientConfig = &tls.Config{ // Can't use SSLv3 because of POODLE and BEAST // Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher // Can't use TLSv1.1 because of RC4 cipher usage MinVersion: tls.VersionTLS12, } - tr.TLSClientConfig = tlsConfig - - // Because we create a custom TLSClientConfig, we have to opt-in to HTTP/2. - // See https://github.com/golang/go/issues/14275 - if err := http2.ConfigureTransport(tr); err != nil { - return nil, err - } } return tr, nil } From 4869c1b1fcad26e49091943269fc1c0699d0c480 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Thu, 23 Jan 2020 02:00:50 +0000 Subject: [PATCH 100/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 235407290..c1d4ec388 100644 --- a/api.go +++ b/api.go @@ -104,7 +104,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.45" + libraryVersion = "v6.0.46" ) // User Agent should always following the below style. From 60e533ec9a34ad510d05d8d4133a43ad8208b654 Mon Sep 17 00:00:00 2001 From: kannappanr <30541348+kannappanr@users.noreply.github.com> Date: Thu, 23 Jan 2020 10:01:36 -0800 Subject: [PATCH 101/215] Add new Amazon S3 endpoints (#1220) Add me-south-1, ap-northeast-3 regions to the list fixes #1184 --- docs/API.md | 16 +++++++++++++++- docs/zh_CN/API.md | 16 +++++++++++++++- s3-endpoints.go | 2 ++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/docs/API.md b/docs/API.md index dbd9ecfee..1bb7d99f6 100644 --- a/docs/API.md +++ b/docs/API.md @@ -132,14 +132,28 @@ __Parameters__ |`bucketName` | _string_ | Name of the bucket | | `location` | _string_ | Region where the bucket is to be created. Default value is us-east-1. Other valid values are listed below. Note: When used with minio server, use the region specified in its config file (defaults to us-east-1).| | | |us-east-1 | +| | |us-east-2 | | | |us-west-1 | | | |us-west-2 | +| | |ca-central-1 | | | |eu-west-1 | +| | |eu-west-2 | +| | |eu-west-3 | | | | eu-central-1| +| | | eu-north-1| +| | | ap-east-1| +| | | ap-south-1| | | | ap-southeast-1| -| | | ap-northeast-1| | | | ap-southeast-2| +| | | ap-northeast-1| +| | | ap-northeast-2| +| | | ap-northeast-3| +| | | me-south-1| | | | sa-east-1| +| | | us-gov-west-1| +| | | us-gov-east-1| +| | | cn-north-1| +| | | cn-northwest-1| __Example__ diff --git a/docs/zh_CN/API.md b/docs/zh_CN/API.md index 1e478973c..eb81b6523 100644 --- a/docs/zh_CN/API.md +++ b/docs/zh_CN/API.md @@ -109,14 +109,28 @@ __参数__ |`bucketName` | _string_ | 存储桶名称 | | `location` | _string_ | 存储桶被创建的region(地区),默认是us-east-1(美国东一区),下面列举的是其它合法的值。注意:如果用的是minio服务的话,resion是在它的配置文件中,(默认是us-east-1)。| | | |us-east-1 | +| | |us-east-2 | | | |us-west-1 | | | |us-west-2 | +| | |ca-central-1 | | | |eu-west-1 | +| | |eu-west-2 | +| | |eu-west-3 | | | | eu-central-1| +| | | eu-north-1| +| | | ap-east-1| +| | | ap-south-1| | | | ap-southeast-1| -| | | ap-northeast-1| | | | ap-southeast-2| +| | | ap-northeast-1| +| | | ap-northeast-2| +| | | ap-northeast-3| +| | | me-south-1| | | | sa-east-1| +| | | us-gov-west-1| +| | | us-gov-east-1| +| | | cn-north-1| +| | | cn-northwest-1| __示例__ diff --git a/s3-endpoints.go b/s3-endpoints.go index 989f58c7e..4a8879fd5 100644 --- a/s3-endpoints.go +++ b/s3-endpoints.go @@ -35,6 +35,8 @@ var awsS3EndpointMap = map[string]string{ "ap-southeast-2": "s3.dualstack.ap-southeast-2.amazonaws.com", "ap-northeast-1": "s3.dualstack.ap-northeast-1.amazonaws.com", "ap-northeast-2": "s3.dualstack.ap-northeast-2.amazonaws.com", + "ap-northeast-3": "s3.dualstack.ap-northeast-3.amazonaws.com", + "me-south-1": "s3.dualstack.me-south-1.amazonaws.com", "sa-east-1": "s3.dualstack.sa-east-1.amazonaws.com", "us-gov-west-1": "s3.dualstack.us-gov-west-1.amazonaws.com", "us-gov-east-1": "s3.dualstack.us-gov-east-1.amazonaws.com", From 2be326ba34a4f6fab86beb0187c85c38f55979b3 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 24 Jan 2020 09:32:26 -0800 Subject: [PATCH 102/215] fix: retry AccessDenied only if Region is present (#1221) --- api.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/api.go b/api.go index c1d4ec388..e75e63b03 100644 --- a/api.go +++ b/api.go @@ -669,20 +669,23 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque case "InvalidRegion": fallthrough case "AccessDenied": - if metadata.bucketName != "" && errResponse.Region != "" { + if errResponse.Region == "" { + // Region is empty we simply return the error. + return res, err + } + // Region is not empty figure out a way to + // handle this appropriately. + if metadata.bucketName != "" { // Gather Cached location only if bucketName is present. if _, cachedOk := c.bucketLocCache.Get(metadata.bucketName); cachedOk { c.bucketLocCache.Set(metadata.bucketName, errResponse.Region) continue // Retry. } } else { - // Most probably for ListBuckets() + // This is for ListBuckets() fallback. if errResponse.Region != metadata.bucketLocation { - // Retry if the error - // response has a - // different region - // than the request we - // just made. + // Retry if the error response has a different region + // than the request we just made. metadata.bucketLocation = errResponse.Region continue // Retry } From a8fa81f2c36397fc9dcc622f4a121776b1f15312 Mon Sep 17 00:00:00 2001 From: poornas Date: Tue, 28 Jan 2020 01:15:32 -0800 Subject: [PATCH 103/215] fix testListObjects functional test (#1222) Remove test for invalid xml character \x17 in object name as it fails with azure gateway storage client on Mint during response parsing because of the invalid xml character in object name. --- functional_tests.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/functional_tests.go b/functional_tests.go index fb66f06fe..d1ec0150c 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -10067,8 +10067,6 @@ func testListObjects() { name string storageClass string }{ - // \x17 is a forbidden character in a xml document - {"foo\x17bar", "STANDARD"}, // Special characters {"foo bar", "STANDARD"}, {"foo-%", "STANDARD"}, From 04b7146507533a42da9f35ec9d82cbbcf89d1e3c Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Thu, 30 Jan 2020 18:46:25 +0000 Subject: [PATCH 104/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index e75e63b03..c44f53c1c 100644 --- a/api.go +++ b/api.go @@ -104,7 +104,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.46" + libraryVersion = "v6.0.47" ) // User Agent should always following the below style. From 1e1c1e727b7c228e267134e79da35008183f5299 Mon Sep 17 00:00:00 2001 From: radix-aw <55851000+radix-aw@users.noreply.github.com> Date: Sat, 1 Feb 2020 12:18:55 -0500 Subject: [PATCH 105/215] Set IAM endpoint to default value if unspecified (#1224) --- pkg/credentials/iam_aws.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/credentials/iam_aws.go b/pkg/credentials/iam_aws.go index 6c6be6659..1b67cbfad 100644 --- a/pkg/credentials/iam_aws.go +++ b/pkg/credentials/iam_aws.go @@ -168,6 +168,10 @@ type ec2RoleCredRespBody struct { // be sent to fetch the rolling access credentials. // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html func getIAMRoleURL(endpoint string) (*url.URL, error) { + if endpoint == "" { + endpoint = defaultIAMRoleEndpoint + } + u, err := url.Parse(endpoint) if err != nil { return nil, err From fc1f142e39c751e9acdde0310d80ce1238613719 Mon Sep 17 00:00:00 2001 From: ebozduman Date: Sat, 1 Feb 2020 10:52:31 -0800 Subject: [PATCH 106/215] Add BucketEncryption apis (#1217) --- README.md | 5 ++ api-get-bucket-encryption.go | 71 +++++++++++++++++ api-put-bucket.go | 97 ++++++++++++++++++++++- docs/API.md | 109 +++++++++++++++++++++++++- examples/s3/deletebucketencryption.go | 49 ++++++++++++ examples/s3/getbucketencryption.go | 52 ++++++++++++ examples/s3/setbucketencryption.go | 58 ++++++++++++++ 7 files changed, 437 insertions(+), 4 deletions(-) create mode 100644 api-get-bucket-encryption.go create mode 100644 examples/s3/deletebucketencryption.go create mode 100644 examples/s3/getbucketencryption.go create mode 100644 examples/s3/setbucketencryption.go diff --git a/README.md b/README.md index 810a93941..33eb6c44d 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,11 @@ The full API Reference is available here. * [setbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketlifecycle.go) * [getbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketlifecycle.go) +### Full Examples : Bucket encryption Operations +* [setbucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketencryption.go) +* [getbucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketencryption.go) +* [deletebucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/deletebucketencryption.go) + ### Full Examples : Bucket notification Operations * [setbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketnotification.go) * [getbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketnotification.go) diff --git a/api-get-bucket-encryption.go b/api-get-bucket-encryption.go new file mode 100644 index 000000000..213abb8fc --- /dev/null +++ b/api-get-bucket-encryption.go @@ -0,0 +1,71 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "encoding/xml" + "io/ioutil" + "net/http" + "net/url" + + "github.com/minio/minio-go/v6/pkg/s3utils" +) + +// GetBucketEncryption - get default encryption configuration for a bucket. +func (c Client) GetBucketEncryption(bucketName string) (ServerSideEncryptionConfiguration, error) { + return c.GetBucketEncryptionWithContext(context.Background(), bucketName) +} + +// GetBucketEncryptionWithContext gets the default encryption configuration on an existing bucket with a context to control cancellations and timeouts. +func (c Client) GetBucketEncryptionWithContext(ctx context.Context, bucketName string) (ServerSideEncryptionConfiguration, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return ServerSideEncryptionConfiguration{}, err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("encryption", "") + + // Execute GET on bucket to get the default encryption configuration. + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + }) + + defer closeResponse(resp) + if err != nil { + return ServerSideEncryptionConfiguration{}, err + } + + if resp.StatusCode != http.StatusOK { + return ServerSideEncryptionConfiguration{}, httpRespToErrorResponse(resp, bucketName, "") + } + + bucketEncryptionBuf, err := ioutil.ReadAll(resp.Body) + if err != nil { + return ServerSideEncryptionConfiguration{}, err + } + + encryptionConfig := ServerSideEncryptionConfiguration{} + if err := xml.Unmarshal(bucketEncryptionBuf, &encryptionConfig); err != nil { + return ServerSideEncryptionConfiguration{}, err + } + return encryptionConfig, nil +} diff --git a/api-put-bucket.go b/api-put-bucket.go index 68371986a..8d9671b1e 100644 --- a/api-put-bucket.go +++ b/api-put-bucket.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,6 +29,25 @@ import ( "github.com/minio/minio-go/v6/pkg/s3utils" ) +// ApplyServerSideEncryptionByDefault defines default encryption configuration, KMS or SSE. To activate +// KMS, SSEAlgoritm needs to be set to "aws:kms" +// Minio currently does not support Kms. +type ApplyServerSideEncryptionByDefault struct { + KmsMasterKeyID string `xml:"KMSMasterKeyID,omitempty"` + SSEAlgorithm string `xml:"SSEAlgorithm"` +} + +// Rule layer encapsulates default encryption configuration +type Rule struct { + Apply ApplyServerSideEncryptionByDefault `xml:"ApplyServerSideEncryptionByDefault"` +} + +// ServerSideEncryptionConfiguration is the default encryption configuration structure +type ServerSideEncryptionConfiguration struct { + XMLName xml.Name `xml:"ServerSideEncryptionConfiguration"` + Rules []Rule `xml:"Rule"` +} + /// Bucket operations func (c Client) makeBucket(ctx context.Context, bucketName string, location string, objectLockEnabled bool) (err error) { @@ -313,6 +332,82 @@ func (c Client) removeBucketLifecycle(ctx context.Context, bucketName string) er return nil } +// SetBucketEncryption sets the default encryption configuration on an existing bucket. +func (c Client) SetBucketEncryption(bucketName string, configuration ServerSideEncryptionConfiguration) error { + return c.SetBucketEncryptionWithContext(context.Background(), bucketName, configuration) +} + +// SetBucketEncryptionWithContext sets the default encryption configuration on an existing bucket with a context to control cancellations and timeouts. +func (c Client) SetBucketEncryptionWithContext(ctx context.Context, bucketName string, configuration ServerSideEncryptionConfiguration) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + buf, err := xml.Marshal(&configuration) + if err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("encryption", "") + + // Content-length is mandatory to set a default encryption configuration + reqMetadata := requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: bytes.NewReader(buf), + contentLength: int64(len(buf)), + contentMD5Base64: sumMD5Base64(buf), + } + + // Execute PUT to upload a new bucket default encryption configuration. + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, bucketName, "") + } + return nil +} + +// DeleteBucketEncryption removes the default encryption configuration on a bucket. +func (c Client) DeleteBucketEncryption(bucketName string) error { + return c.DeleteBucketEncryptionWithContext(context.Background(), bucketName) +} + +// DeleteBucketEncryptionWithContext removes the default encryption configuration on a bucket with a context to control cancellations and timeouts. +func (c Client) DeleteBucketEncryptionWithContext(ctx context.Context, bucketName string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("encryption", "") + + // DELETE default encryption configuration on a bucket. + resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return httpRespToErrorResponse(resp, bucketName, "") + } + return nil +} + // SetBucketNotification saves a new bucket notification. func (c Client) SetBucketNotification(bucketName string, bucketNotification BucketNotification) error { return c.SetBucketNotificationWithContext(context.Background(), bucketName, bucketNotification) diff --git a/docs/API.md b/docs/API.md index 1bb7d99f6..dec6d445b 100644 --- a/docs/API.md +++ b/docs/API.md @@ -64,9 +64,9 @@ func main() { | | [`ComposeObject`](#ComposeObject) | [`ComposeObject`](#ComposeObject) | | [`GetBucketObjectLockConfig`](#GetBucketObjectLockConfig) | | | | [`NewSourceInfo`](#NewSourceInfo) | [`NewSourceInfo`](#NewSourceInfo) | | [`EnableVersioning`](#EnableVersioning) | | | | [`NewDestinationInfo`](#NewDestinationInfo) | [`NewDestinationInfo`](#NewDestinationInfo) | | [`DisableVersioning`](#DisableVersioning) | | -| | [`PutObjectWithContext`](#PutObjectWithContext) | [`PutObjectWithContext`](#PutObjectWithContext) | | | -| | [`GetObjectWithContext`](#GetObjectWithContext) | [`GetObjectWithContext`](#GetObjectWithContext) | | | -| | [`FPutObjectWithContext`](#FPutObjectWithContext) | [`FPutObjectWithContext`](#FPutObjectWithContext) | | | +| | [`PutObjectWithContext`](#PutObjectWithContext) | [`PutObjectWithContext`](#PutObjectWithContext) | | [`SetBucketEncryption`](#SetBucketEncryption) | +| | [`GetObjectWithContext`](#GetObjectWithContext) | [`GetObjectWithContext`](#GetObjectWithContext) | | [`GetBucketEncryption`](#GetBucketEncryption) | | +| | [`FPutObjectWithContext`](#FPutObjectWithContext) | [`FPutObjectWithContext`](#FPutObjectWithContext) | | [`DeleteBucketEncryption`](#DeleteBucketEncryption) | | | [`FGetObjectWithContext`](#FGetObjectWithContext) | [`FGetObjectWithContext`](#FGetObjectWithContext) | | | | | [`RemoveObjectsWithContext`](#RemoveObjectsWithContext) | | | | | | [`RemoveObjectWithOptions`](#RemoveObjectWithOptions) | | | | @@ -1920,6 +1920,109 @@ if err != nil { } ``` + +### SetBucketEncryption(bucketname string, configuration ServerSideEncryptionConfiguration) error +Set default encryption configuration on a bucket. + +__Parameters__ + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket| +|`configuration` | _minio.ServerSideEncyrptionConfiguration_ | Structure that holds default encryption configuration to be set | + +__Return Values__ + +|Param |Type |Description | +|:---|:---| :---| +|`err` | _error_ |Standard Error | + +__Example__ + +```go +s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) +if err != nil { + log.Fatalln(err) +} + +// Initialize default encryption configuration structure +config := minio.ServerSideEncryptionConfiguration{Rules: []minio.Rule{ + minio.Rule{ + Apply: minio.ApplyServerSideEncryptionByDefault{ + SSEAlgorithm: "AES256", + }, + }, +}} +// Set default encryption configuration on an S3 bucket +err = s3Client.SetBucketEncryption("my-bucketname", config) +if err != nil { + log.Fatalln(err) +} +``` + + +### GetBucketEncryption(bucketName string) (ServerSideEncryptionConfiguration, error) +Get default encryption configuration set on a bucket. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | + +__Return Values__ + + +|Param |Type |Description | +|:---|:---| :---| +|`configuration` | _minio.ServerSideEncyrptionConfiguration_ | Structure that holds default encryption configuration | +|`err` | _error_ |Standard Error | + +__Example__ + +```go +s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) +if err != nil { + log.Fatalln(err) +} + +// Get default encryption configuration set on an S3 bucket and print it out +encryptionConfig, err := s3Client.GetBucketEncryption("my-bucketname") +if err != nil { + log.Fatalln(err) +} +fmt.Printf("%+v\n", encryptionConfig) +``` + + +### DeleteBucketEncryption(bucketName string) (error) +Delete/Remove default encryption configuration set on a bucket. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---|:---| +|`bucketName` | _string_ |Name of the bucket | + +__Return Values__ + + +|Param |Type |Description | +|:---|:---| :---| +|`err` | _error_ |Standard Error | + +__Example__ + +```go +err := s3Client.DeleteBucketEncryption("my-bucketname") +if err != nil { + log.Fatalln(err) +} +// "my-bucket" is successfully deleted/removed. +``` + ### SetBucketObjectLockConfig(bucketname, mode *RetentionMode, validity *uint, unit *ValidityUnit) error Set object lock configuration in given bucket. mode, validity and unit are either all set or all nil. diff --git a/examples/s3/deletebucketencryption.go b/examples/s3/deletebucketencryption.go new file mode 100644 index 000000000..f050bc230 --- /dev/null +++ b/examples/s3/deletebucketencryption.go @@ -0,0 +1,49 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // s3Client.TraceOn(os.Stderr) + + // Get default encryption configuration set on a S3 bucket + err = s3Client.DeleteBucketEncryption("my-bucketname") + if err != nil { + log.Fatalln(err) + } +} diff --git a/examples/s3/getbucketencryption.go b/examples/s3/getbucketencryption.go new file mode 100644 index 000000000..ef47d5797 --- /dev/null +++ b/examples/s3/getbucketencryption.go @@ -0,0 +1,52 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + "log" + + "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // s3Client.TraceOn(os.Stderr) + + // Get default encryption configuration set on an S3 bucket, + // and print out the encryption configuration. + encryptionConfig, err := s3Client.GetBucketEncryption("my-bucketname") + if err != nil { + log.Fatalln(err) + } + fmt.Printf("%+v\n", encryptionConfig) +} diff --git a/examples/s3/setbucketencryption.go b/examples/s3/setbucketencryption.go new file mode 100644 index 000000000..85f7a49b2 --- /dev/null +++ b/examples/s3/setbucketencryption.go @@ -0,0 +1,58 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // s3Client.TraceOn(os.Stderr) + + // Set default encryption configuration on a bucket + config := minio.ServerSideEncryptionConfiguration{Rules: []minio.Rule{ + minio.Rule{ + Apply: minio.ApplyServerSideEncryptionByDefault{ + SSEAlgorithm: "AES256", + // KmsMasterKeyID: "my-masterkey", + // SSEAlgorithm: "aws:kms", + }, + }, + }} + err = s3Client.SetBucketEncryption("my-bucketname", config) + if err != nil { + log.Fatalln(err) + } +} From f288d9cee2f9fd96f09b12392400a540a36ef4c5 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Thu, 6 Feb 2020 19:21:51 +0000 Subject: [PATCH 107/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index c44f53c1c..15b80d3f5 100644 --- a/api.go +++ b/api.go @@ -104,7 +104,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.47" + libraryVersion = "v6.0.48" ) // User Agent should always following the below style. From e2cfe96bf296f4210500566280aebcef6b57d61e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Wed, 12 Feb 2020 00:38:32 +0200 Subject: [PATCH 108/215] Bucket and object name length error message fixes (#1227) --- pkg/s3utils/utils.go | 6 +++--- pkg/s3utils/utils_test.go | 4 ++-- utils_test.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/s3utils/utils.go b/pkg/s3utils/utils.go index f1f6eea60..988bf8d3a 100644 --- a/pkg/s3utils/utils.go +++ b/pkg/s3utils/utils.go @@ -299,10 +299,10 @@ func checkBucketNameCommon(bucketName string, strict bool) (err error) { return errors.New("Bucket name cannot be empty") } if len(bucketName) < 3 { - return errors.New("Bucket name cannot be smaller than 3 characters") + return errors.New("Bucket name cannot be shorter than 3 characters") } if len(bucketName) > 63 { - return errors.New("Bucket name cannot be greater than 63 characters") + return errors.New("Bucket name cannot be longer than 63 characters") } if ipAddress.MatchString(bucketName) { return errors.New("Bucket name cannot be an ip address") @@ -338,7 +338,7 @@ func CheckValidBucketNameStrict(bucketName string) (err error) { // - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html func CheckValidObjectNamePrefix(objectName string) error { if len(objectName) > 1024 { - return errors.New("Object name cannot be greater than 1024 characters") + return errors.New("Object name cannot be longer than 1024 characters") } if !utf8.ValidString(objectName) { return errors.New("Object name with non UTF-8 strings are not supported") diff --git a/pkg/s3utils/utils_test.go b/pkg/s3utils/utils_test.go index 587b5cd32..2e5e66cff 100644 --- a/pkg/s3utils/utils_test.go +++ b/pkg/s3utils/utils_test.go @@ -333,7 +333,7 @@ func TestIsValidBucketName(t *testing.T) { {".mybucket", errors.New("Bucket name contains invalid characters"), false}, {"$mybucket", errors.New("Bucket name contains invalid characters"), false}, {"mybucket-", errors.New("Bucket name contains invalid characters"), false}, - {"my", errors.New("Bucket name cannot be smaller than 3 characters"), false}, + {"my", errors.New("Bucket name cannot be shorter than 3 characters"), false}, {"", errors.New("Bucket name cannot be empty"), false}, {"my..bucket", errors.New("Bucket name contains invalid characters"), false}, {"my.-bucket", errors.New("Bucket name contains invalid characters"), false}, @@ -381,7 +381,7 @@ func TestIsValidBucketNameStrict(t *testing.T) { {".mybucket", errors.New("Bucket name contains invalid characters"), false}, {"$mybucket", errors.New("Bucket name contains invalid characters"), false}, {"mybucket-", errors.New("Bucket name contains invalid characters"), false}, - {"my", errors.New("Bucket name cannot be smaller than 3 characters"), false}, + {"my", errors.New("Bucket name cannot be shorter than 3 characters"), false}, {"", errors.New("Bucket name cannot be empty"), false}, {"my..bucket", errors.New("Bucket name contains invalid characters"), false}, {"my.-bucket", errors.New("Bucket name contains invalid characters"), false}, diff --git a/utils_test.go b/utils_test.go index 0ce163812..03a0a82d5 100644 --- a/utils_test.go +++ b/utils_test.go @@ -273,7 +273,7 @@ func TestIsValidBucketName(t *testing.T) { {".mybucket", ErrInvalidBucketName("Bucket name contains invalid characters"), false}, {"mybucket.", ErrInvalidBucketName("Bucket name contains invalid characters"), false}, {"mybucket-", ErrInvalidBucketName("Bucket name contains invalid characters"), false}, - {"my", ErrInvalidBucketName("Bucket name cannot be smaller than 3 characters"), false}, + {"my", ErrInvalidBucketName("Bucket name cannot be shorter than 3 characters"), false}, {"", ErrInvalidBucketName("Bucket name cannot be empty"), false}, {"my..bucket", ErrInvalidBucketName("Bucket name contains invalid characters"), false}, {"my.bucket.com", nil, true}, From 3d72010f0363469ae7593194422fa50d15210215 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 14 Feb 2020 11:44:17 +0530 Subject: [PATCH 109/215] fix: ignore AWS elb endpoints in region extraction (#1228) --- pkg/s3utils/utils.go | 18 ++++++++++++++---- pkg/s3utils/utils_test.go | 17 +++++++++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/pkg/s3utils/utils.go b/pkg/s3utils/utils.go index 988bf8d3a..982dddf4b 100644 --- a/pkg/s3utils/utils.go +++ b/pkg/s3utils/utils.go @@ -84,16 +84,22 @@ func IsVirtualHostSupported(endpointURL url.URL, bucketName string) bool { // Refer for region styles - https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region // amazonS3HostHyphen - regular expression used to determine if an arg is s3 host in hyphenated style. -var amazonS3HostHyphen = regexp.MustCompile(`^s3-(.*?)\.amazonaws\.com$`) +var amazonS3HostHyphen = regexp.MustCompile(`^s3-(.*?).amazonaws.com$`) // amazonS3HostDualStack - regular expression used to determine if an arg is s3 host dualstack. -var amazonS3HostDualStack = regexp.MustCompile(`^s3\.dualstack\.(.*?)\.amazonaws\.com$`) +var amazonS3HostDualStack = regexp.MustCompile(`^s3.dualstack.(.*?).amazonaws.com$`) // amazonS3HostDot - regular expression used to determine if an arg is s3 host in . style. -var amazonS3HostDot = regexp.MustCompile(`^s3\.(.*?)\.amazonaws\.com$`) +var amazonS3HostDot = regexp.MustCompile(`^s3.(.*?).amazonaws.com$`) // amazonS3ChinaHost - regular expression used to determine if the arg is s3 china host. -var amazonS3ChinaHost = regexp.MustCompile(`^s3\.(cn.*?)\.amazonaws\.com\.cn$`) +var amazonS3ChinaHost = regexp.MustCompile(`^s3.(cn.*?).amazonaws.com.cn$`) + +// Regular expression used to determine if the arg is elb host. +var elbAmazonRegex = regexp.MustCompile(`elb(.*?).amazonaws.com$`) + +// Regular expression used to determine if the arg is elb host in china. +var elbAmazonCnRegex = regexp.MustCompile(`elb(.*?).amazonaws.com.cn$`) // GetRegionFromURL - returns a region from url host. func GetRegionFromURL(endpointURL url.URL) string { @@ -106,6 +112,10 @@ func GetRegionFromURL(endpointURL url.URL) string { if IsAmazonGovCloudEndpoint(endpointURL) { return "us-gov-west-1" } + // if elb's are used we cannot calculate which region it may be, just return empty. + if elbAmazonRegex.MatchString(endpointURL.Host) || elbAmazonCnRegex.MatchString(endpointURL.Host) { + return "" + } parts := amazonS3HostDualStack.FindStringSubmatch(endpointURL.Host) if len(parts) > 1 { return parts[1] diff --git a/pkg/s3utils/utils_test.go b/pkg/s3utils/utils_test.go index 2e5e66cff..e2fd3089d 100644 --- a/pkg/s3utils/utils_test.go +++ b/pkg/s3utils/utils_test.go @@ -73,6 +73,23 @@ func TestGetRegionFromURL(t *testing.T) { u: url.URL{Host: "s3-external-1.amazonaws.com"}, expectedRegion: "", }, + { + u: url.URL{ + Host: "s3.kubernetesfrontendlb-caf78da2b1f7516c.elb.us-west-2.amazonaws.com", + }, + expectedRegion: "", + }, + { + u: url.URL{ + Host: "s3.kubernetesfrontendlb-caf78da2b1f7516c.elb.amazonaws.com", + }, + expectedRegion: "", + }, + { + u: url.URL{ + Host: "s3.kubernetesfrontendlb-caf78da2b1f7516c.elb.amazonaws.com.cn", + }, + }, } for i, testCase := range testCases { From e1b170b7f4d3f63ca004dcc57bf189989add1910 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Fri, 14 Feb 2020 18:02:08 +0000 Subject: [PATCH 110/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 15b80d3f5..d9fc59a51 100644 --- a/api.go +++ b/api.go @@ -104,7 +104,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.48" + libraryVersion = "v6.0.49" ) // User Agent should always following the below style. From 112c09f43c7893c6d37bff3259840cb428a51130 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 18 Feb 2020 21:28:44 +0530 Subject: [PATCH 111/215] extract userMetadata from event response (#1229) --- api-notification.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/api-notification.go b/api-notification.go index 0480c21eb..b08ad993a 100644 --- a/api-notification.go +++ b/api-notification.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 MinIO, Inc. + * Copyright 2017-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -89,11 +89,13 @@ type bucketMeta struct { // Notification event object metadata. type objectMeta struct { - Key string `json:"key"` - Size int64 `json:"size,omitempty"` - ETag string `json:"eTag,omitempty"` - VersionID string `json:"versionId,omitempty"` - Sequencer string `json:"sequencer"` + Key string `json:"key"` + Size int64 `json:"size,omitempty"` + ETag string `json:"eTag,omitempty"` + ContentType string `json:"contentType,omitempty"` + UserMetadata map[string]string `json:"userMetadata,omitempty"` + VersionID string `json:"versionId,omitempty"` + Sequencer string `json:"sequencer"` } // Notification event server specific metadata. From 8730ce1d61167ac1af7ad21ef13493895738390d Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Wed, 19 Feb 2020 00:16:23 +0530 Subject: [PATCH 112/215] Add LegalHold API Support (#1226) --- api-object-legal-hold.go | 176 ++++++++++++++++++++++++++++++ docs/API.md | 53 +++++++++ examples/s3/getobjectlegalhold.go | 49 +++++++++ examples/s3/putobjectlegalhold.go | 50 +++++++++ 4 files changed, 328 insertions(+) create mode 100644 api-object-legal-hold.go create mode 100644 examples/s3/getobjectlegalhold.go create mode 100644 examples/s3/putobjectlegalhold.go diff --git a/api-object-legal-hold.go b/api-object-legal-hold.go new file mode 100644 index 000000000..2b43575ac --- /dev/null +++ b/api-object-legal-hold.go @@ -0,0 +1,176 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "context" + "encoding/xml" + "fmt" + "net/http" + "net/url" + + "github.com/minio/minio-go/v6/pkg/s3utils" +) + +// objectLegalHold - object legal hold specified in +// https://docs.aws.amazon.com/AmazonS3/latest/API/archive-RESTObjectPUTLegalHold.html +type objectLegalHold struct { + XMLNS string `xml:"xmlns,attr,omitempty"` + XMLName xml.Name `xml:"LegalHold"` + Status LegalHoldStatus `xml:"Status,omitempty"` +} + +// PutObjectLegalHoldOptions represents options specified by user for PutObjectLegalHold call +type PutObjectLegalHoldOptions struct { + VersionID string + Status *LegalHoldStatus +} + +// GetObjectLegalHoldOptions represents options specified by user for GetObjectLegalHold call +type GetObjectLegalHoldOptions struct { + VersionID string +} + +// LegalHoldStatus - object legal hold status. +type LegalHoldStatus string + +const ( + // LegalHoldEnabled indicates legal hold is enabled + LegalHoldEnabled LegalHoldStatus = "ON" + + // LegalHoldDisabled indicates legal hold is disabled + LegalHoldDisabled LegalHoldStatus = "OFF" +) + +func (r LegalHoldStatus) String() string { + return string(r) +} + +// IsValid - check whether this legal hold status is valid or not. +func (r LegalHoldStatus) IsValid() bool { + return r == LegalHoldEnabled || r == LegalHoldDisabled +} + +func newObjectLegalHold(status *LegalHoldStatus) (*objectLegalHold, error) { + if status == nil { + return nil, fmt.Errorf("Status not set") + } + if !status.IsValid() { + return nil, fmt.Errorf("invalid legal hold status `%v`", status) + } + legalHold := &objectLegalHold{ + Status: *status, + } + return legalHold, nil +} + +// PutObjectLegalHold : sets object legal hold for a given object and versionID. +func (c Client) PutObjectLegalHold(bucketName, objectName string, opts PutObjectLegalHoldOptions) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("legal-hold", "") + + if opts.VersionID != "" { + urlValues.Set("versionId", opts.VersionID) + } + + lh, err := newObjectLegalHold(opts.Status) + if err != nil { + return err + } + + lhData, err := xml.Marshal(lh) + if err != nil { + return err + } + + reqMetadata := requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: urlValues, + contentBody: bytes.NewReader(lhData), + contentLength: int64(len(lhData)), + contentMD5Base64: sumMD5Base64(lhData), + contentSHA256Hex: sum256Hex(lhData), + } + + // Execute PUT Object Legal Hold. + resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp != nil { + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return httpRespToErrorResponse(resp, bucketName, objectName) + } + } + return nil +} + +// GetObjectLegalHold gets legal-hold status of given object. +func (c Client) GetObjectLegalHold(bucketName, objectName string, opts GetObjectLegalHoldOptions) (status *LegalHoldStatus, err error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return nil, err + } + + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return nil, err + } + urlValues := make(url.Values) + urlValues.Set("legal-hold", "") + + if opts.VersionID != "" { + urlValues.Set("versionId", opts.VersionID) + } + + // Execute GET on bucket to list objects. + resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return nil, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return nil, httpRespToErrorResponse(resp, bucketName, objectName) + } + } + lh := &objectLegalHold{} + if err = xml.NewDecoder(resp.Body).Decode(lh); err != nil { + return nil, err + } + + return &lh.Status, nil +} diff --git a/docs/API.md b/docs/API.md index dec6d445b..0f80a0d70 100644 --- a/docs/API.md +++ b/docs/API.md @@ -72,6 +72,8 @@ func main() { | | [`RemoveObjectWithOptions`](#RemoveObjectWithOptions) | | | | | | [`PutObjectRetention`](#PutObjectRetention) | | | | | | [`GetObjectRetention`](#GetObjectRetention) | | | | +| | [`PutObjectLegalHold`](#PutObjectLegalHold) | | | | +| | [`GetObjectLegalHold`](#GetObjectLegalHold) | | | | | | [`SelectObjectContent`](#SelectObjectContent) | | | | [`PutObjectTagging`](#PutObjectTagging) | | | | [`PutObjectTaggingWithContext`](#PutObjectTaggingWithContext) | | @@ -1279,6 +1281,57 @@ if err != nil { return } ``` + +### PutObjectLegalHold(bucketName, objectName string, opts minio.PutObjectLegalHoldOptions) error +Applies legal-hold onto an object. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`opts` |_minio.PutObjectLegalHoldOptions_ |Allows user to set options like status and version id | + +_minio.PutObjectLegalHoldOptions_ + +|Field | Type | Description | +|:--- |:--- | :--- | +| `opts.Status` | _*minio.LegalHoldStatus_ |Legal-Hold status to be set| +| `opts.VersionID` | _string_ |Version ID of the object to apply retention on| + +```go +s := minio.LegalHoldEnabled +opts := minio.PutObjectLegalHoldOptions { + Status: &s, +} +err = minioClient.PutObjectLegalHold("mybucket", "myobject", opts) +if err != nil { + fmt.Println(err) + return +} +``` + +### GetObjectLegalHold(bucketName, objectName, versionID string) (status *LegalHoldStatus, err error) +Returns legal-hold status on a given object. + +__Parameters__ + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`opts` |_minio.GetObjectLegalHoldOptions_ |Allows user to set options like version id | + +```go +opts := minio.GetObjectLegalHoldOptions{} +err = minioClient.GetObjectLegalHold("mybucket", "myobject", opts) +if err != nil { + fmt.Println(err) + return +} +``` ### SelectObjectContent(ctx context.Context, bucketName string, objectsName string, expression string, options SelectObjectOptions) *SelectResults Parameters diff --git a/examples/s3/getobjectlegalhold.go b/examples/s3/getobjectlegalhold.go new file mode 100644 index 000000000..1ac7b7bdb --- /dev/null +++ b/examples/s3/getobjectlegalhold.go @@ -0,0 +1,49 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + "log" + + "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and + // my-testfile are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + if err != nil { + log.Fatalln(err) + } + opts := minio.GetObjectLegalHoldOptions{} + lh, err := s3Client.GetObjectLegalHold("my-bucket", "my-object", opts) + if err != nil { + log.Fatalln(err) + } + fmt.Printf("Legal Hold on object is %s", lh) + log.Println("Get object legal-hold on my-object successfully.") +} diff --git a/examples/s3/putobjectlegalhold.go b/examples/s3/putobjectlegalhold.go new file mode 100644 index 000000000..93cd43586 --- /dev/null +++ b/examples/s3/putobjectlegalhold.go @@ -0,0 +1,50 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and + // my-testfile are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + if err != nil { + log.Fatalln(err) + } + s := minio.LegalHoldEnabled + opts := minio.PutObjectLegalHoldOptions{ + Status: &s, + } + err = s3Client.PutObjectLegalHold("my-bucket", "my-object", opts) + if err != nil { + log.Fatalln(err) + } + log.Println("Set object legal-hold on my-object successfully.") +} From f338a19b7ea4af0c01e33cefced26866ae2394ac Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Thu, 20 Feb 2020 23:41:10 +0000 Subject: [PATCH 113/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index d9fc59a51..0898d69c3 100644 --- a/api.go +++ b/api.go @@ -104,7 +104,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.49" + libraryVersion = "v6.0.50" ) // User Agent should always following the below style. From 8c915c0c0bf227048b9fe24af65525a43bd784cc Mon Sep 17 00:00:00 2001 From: kannappanr <30541348+kannappanr@users.noreply.github.com> Date: Wed, 4 Mar 2020 16:54:49 -0800 Subject: [PATCH 114/215] Add Support for Legal-Hold (#1233) This PR adds support for legal-hold in PutObject, GetObject, StatObject & CopyObject APIs. This PR also adds support for "SendContentMd5" field in PutObjectOptions so users can ask the SDK to send the content-md5. Users are expected to set this flag for PutObject operations in case locking is enabled on given bucket. Note regarding PutObject behaviour: If SendContentMd5 flag is set minio-go may consume higher memory due to in-memory md5sum calculation. --- api-compose-object.go | 4 ++ api-object-lock.go | 14 +++--- api-put-object-copy.go | 4 ++ api-put-object-multipart.go | 4 +- api-put-object-streaming.go | 70 +++++++++++++++++++++++++----- api-put-object.go | 28 +++++++++--- api.go | 5 ++- bucket-notification.go | 2 +- constants.go | 3 ++ docs/API.md | 11 +++++ examples/s3/setbucketencryption.go | 2 +- post-policy.go | 2 +- 12 files changed, 119 insertions(+), 30 deletions(-) diff --git a/api-compose-object.go b/api-compose-object.go index c9e44c556..e534b88e4 100644 --- a/api-compose-object.go +++ b/api-compose-object.go @@ -60,6 +60,9 @@ type DestInfoOptions struct { // Otherwise this field is ignored UserTags map[string]string ReplaceTags bool + + // Specifies whether you want to apply a Legal Hold to the copied object. + LegalHold LegalHoldStatus } // Process custom-metadata to remove a `x-amz-meta-` prefix if @@ -107,6 +110,7 @@ func NewDestinationInfo(bucket, object string, sse encrypt.ServerSide, userMeta UserMeta: m, UserTags: nil, ReplaceTags: false, + LegalHold: LegalHoldStatus(""), } return DestinationInfo{ bucket: bucket, diff --git a/api-object-lock.go b/api-object-lock.go index c30ab3258..fa8b81709 100644 --- a/api-object-lock.go +++ b/api-object-lock.go @@ -184,10 +184,10 @@ func (c Client) SetBucketObjectLockConfig(bucketName string, mode *RetentionMode } // GetBucketObjectLockConfig gets object lock configuration of given bucket. -func (c Client) GetBucketObjectLockConfig(bucketName string) (mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { +func (c Client) GetBucketObjectLockConfig(bucketName string) (objectLock string, mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return nil, nil, nil, err + return "", nil, nil, nil, err } urlValues := make(url.Values) @@ -201,16 +201,16 @@ func (c Client) GetBucketObjectLockConfig(bucketName string) (mode *RetentionMod }) defer closeResponse(resp) if err != nil { - return nil, nil, nil, err + return "", nil, nil, nil, err } if resp != nil { if resp.StatusCode != http.StatusOK { - return nil, nil, nil, httpRespToErrorResponse(resp, bucketName, "") + return "", nil, nil, nil, httpRespToErrorResponse(resp, bucketName, "") } } config := &objectLockConfig{} if err = xml.NewDecoder(resp.Body).Decode(config); err != nil { - return nil, nil, nil, err + return "", nil, nil, nil, err } if config.Rule != nil { @@ -225,8 +225,8 @@ func (c Client) GetBucketObjectLockConfig(bucketName string) (mode *RetentionMod unit = &years } - return mode, validity, unit, nil + return config.ObjectLockEnabled, mode, validity, unit, nil } - return nil, nil, nil, nil + return "", nil, nil, nil, nil } diff --git a/api-put-object-copy.go b/api-put-object-copy.go index ce48674b0..f4db346fb 100644 --- a/api-put-object-copy.go +++ b/api-put-object-copy.go @@ -45,6 +45,10 @@ func (c Client) CopyObjectWithProgress(dst DestinationInfo, src SourceInfo, prog header.Set(amzTaggingHeader, s3utils.TagEncode(dst.opts.UserTags)) } + if dst.opts.LegalHold != LegalHoldStatus("") { + header.Set(amzLegalHoldHeader, dst.opts.LegalHold.String()) + } + var err error var size int64 // If progress bar is specified, size should be requested as well initiate a StatObject request. diff --git a/api-put-object-multipart.go b/api-put-object-multipart.go index 864acc8bc..e5bd37d5d 100644 --- a/api-put-object-multipart.go +++ b/api-put-object-multipart.go @@ -50,7 +50,7 @@ func (c Client) putObjectMultipart(ctx context.Context, bucketName, objectName s return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) } // Fall back to uploading as single PutObject operation. - return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts) + return c.putObject(ctx, bucketName, objectName, reader, size, opts) } } return n, err @@ -104,7 +104,7 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje // Choose hash algorithms to be calculated by hashCopyN, // avoid sha256 with non-v4 signature request or // HTTPS connection. - hashAlgos, hashSums := c.hashMaterials() + hashAlgos, hashSums := c.hashMaterials(opts.SendContentMd5) length, rErr := io.ReadFull(reader, buf) if rErr == io.EOF { diff --git a/api-put-object-streaming.go b/api-put-object-streaming.go index 5add2afcf..9463062a1 100644 --- a/api-put-object-streaming.go +++ b/api-put-object-streaming.go @@ -18,10 +18,14 @@ package minio import ( + "bytes" "context" + "crypto/md5" + "encoding/base64" "fmt" "io" "net/http" + "runtime/debug" "sort" "strings" @@ -40,11 +44,11 @@ import ( func (c Client) putObjectMultipartStream(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) { - if !isObject(reader) && isReadAt(reader) { + if !isObject(reader) && isReadAt(reader) && !opts.SendContentMd5 { // Verify if the reader implements ReadAt and it is not a *minio.Object then we will use parallel uploader. n, err = c.putObjectMultipartStreamFromReadAt(ctx, bucketName, objectName, reader.(io.ReaderAt), size, opts) } else { - n, err = c.putObjectMultipartStreamNoChecksum(ctx, bucketName, objectName, reader, size, opts) + n, err = c.putObjectMultipartStreamOptionalChecksum(ctx, bucketName, objectName, reader, size, opts) } if err != nil { errResp := ToErrorResponse(err) @@ -56,7 +60,7 @@ func (c Client) putObjectMultipartStream(ctx context.Context, bucketName, object return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) } // Fall back to uploading as single PutObject operation. - return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts) + return c.putObject(ctx, bucketName, objectName, reader, size, opts) } } return n, err @@ -220,7 +224,7 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa return totalUploadedSize, nil } -func (c Client) putObjectMultipartStreamNoChecksum(ctx context.Context, bucketName, objectName string, +func (c Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) { // Input validation. if err = s3utils.CheckValidBucketName(bucketName); err != nil { @@ -257,20 +261,47 @@ func (c Client) putObjectMultipartStreamNoChecksum(ctx context.Context, bucketNa // Initialize parts uploaded map. partsInfo := make(map[int]ObjectPart) + // Create a buffer. + buf := make([]byte, partSize) + defer debug.FreeOSMemory() + + // Avoid declaring variables in the for loop + var md5Base64 string + var hookReader io.Reader + // Part number always starts with '1'. var partNumber int for partNumber = 1; partNumber <= totalPartsCount; partNumber++ { - // Update progress reader appropriately to the latest offset - // as we read from the source. - hookReader := newHook(reader, opts.Progress) // Proceed to upload the part. if partNumber == totalPartsCount { partSize = lastPartSize } + + if opts.SendContentMd5 { + length, rerr := io.ReadFull(reader, buf) + if rerr == io.EOF && partNumber > 1 { + break + } + if rerr != nil && rerr != io.ErrUnexpectedEOF && rerr != io.EOF { + return 0, rerr + } + // Calculate md5sum. + hash := md5.New() + hash.Write(buf[:length]) + md5Base64 = base64.StdEncoding.EncodeToString(hash.Sum(nil)) + // Update progress reader appropriately to the latest offset + // as we read from the source. + hookReader = newHook(bytes.NewReader(buf[:length]), opts.Progress) + } else { + // Update progress reader appropriately to the latest offset + // as we read from the source. + hookReader = newHook(reader, opts.Progress) + } + objPart, uerr := c.uploadPart(ctx, bucketName, objectName, uploadID, io.LimitReader(hookReader, partSize), - partNumber, "", "", partSize, opts.ServerSideEncryption) + partNumber, md5Base64, "", partSize, opts.ServerSideEncryption) if uerr != nil { return totalUploadedSize, uerr } @@ -316,9 +347,9 @@ func (c Client) putObjectMultipartStreamNoChecksum(ctx context.Context, bucketNa return totalUploadedSize, nil } -// putObjectNoChecksum special function used Google Cloud Storage. This special function +// putObject special function used Google Cloud Storage. This special function // is used for Google Cloud Storage since Google's multipart API is not S3 compatible. -func (c Client) putObjectNoChecksum(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) { +func (c Client) putObject(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return 0, err @@ -343,13 +374,30 @@ func (c Client) putObjectNoChecksum(ctx context.Context, bucketName, objectName } } + var md5Base64 string + if opts.SendContentMd5 { + // Create a buffer. + buf := make([]byte, size) + defer debug.FreeOSMemory() + + length, rErr := io.ReadFull(reader, buf) + if rErr != nil && rErr != io.ErrUnexpectedEOF { + return 0, rErr + } + + // Calculate md5sum. + hash := md5.New() + hash.Write(buf[:length]) + md5Base64 = base64.StdEncoding.EncodeToString(hash.Sum(nil)) + } + // Update progress reader appropriately to the latest offset as we // read from the source. readSeeker := newHook(reader, opts.Progress) // This function does not calculate sha256 and md5sum for payload. // Execute put object. - st, err := c.putObjectDo(ctx, bucketName, objectName, readSeeker, "", "", size, opts) + st, err := c.putObjectDo(ctx, bucketName, objectName, readSeeker, md5Base64, "", size, opts) if err != nil { return 0, err } diff --git a/api-put-object.go b/api-put-object.go index c760056cb..4598f58ce 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -20,6 +20,8 @@ package minio import ( "bytes" "context" + "crypto/md5" + "encoding/base64" "fmt" "io" "net/http" @@ -49,6 +51,8 @@ type PutObjectOptions struct { StorageClass string WebsiteRedirectLocation string PartSize uint64 + LegalHold LegalHoldStatus + SendContentMd5 bool } // getNumThreads - gets the number of threads to be used in the multipart @@ -92,6 +96,9 @@ func (opts PutObjectOptions) Header() (header http.Header) { header["x-amz-object-lock-retain-until-date"] = []string{opts.RetainUntilDate.Format(time.RFC3339)} } + if opts.LegalHold != "" { + header[amzLegalHoldHeader] = []string{opts.LegalHold.String()} + } if opts.ServerSideEncryption != nil { opts.ServerSideEncryption.Marshal(header) } @@ -129,6 +136,9 @@ func (opts PutObjectOptions) validate() (err error) { return ErrInvalidArgument(opts.Mode.String() + " unsupported retention mode") } } + if opts.LegalHold != "" && !opts.LegalHold.IsValid() { + return ErrInvalidArgument(opts.LegalHold.String() + " unsupported legal-hold status") + } return nil } @@ -164,8 +174,7 @@ func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName stri // NOTE: Streaming signature is not supported by GCS. if s3utils.IsGoogleEndpoint(*c.endpointURL) { - // Do not compute MD5 for Google Cloud Storage. - return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts) + return c.putObject(ctx, bucketName, objectName, reader, size, opts) } partSize := opts.PartSize @@ -175,7 +184,7 @@ func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName stri if c.overrideSignerType.IsV2() { if size >= 0 && size < int64(partSize) { - return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts) + return c.putObject(ctx, bucketName, objectName, reader, size, opts) } return c.putObjectMultipart(ctx, bucketName, objectName, reader, size, opts) } @@ -184,10 +193,9 @@ func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName stri } if size < int64(partSize) { - return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts) + return c.putObject(ctx, bucketName, objectName, reader, size, opts) } - // For all sizes greater than 128MiB do multipart. return c.putObjectMultipartStream(ctx, bucketName, objectName, reader, size, opts) } @@ -243,13 +251,21 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName return 0, rerr } + var md5Base64 string + if opts.SendContentMd5 { + // Calculate md5sum. + hash := md5.New() + hash.Write(buf[:length]) + md5Base64 = base64.StdEncoding.EncodeToString(hash.Sum(nil)) + } + // Update progress reader appropriately to the latest offset // as we read from the source. rd := newHook(bytes.NewReader(buf[:length]), opts.Progress) // Proceed to upload the part. objPart, uerr := c.uploadPart(ctx, bucketName, objectName, uploadID, rd, partNumber, - "", "", int64(length), opts.ServerSideEncryption) + md5Base64, "", int64(length), opts.ServerSideEncryption) if uerr != nil { return totalUploadedSize, uerr } diff --git a/api.go b/api.go index 0898d69c3..1a4e0699b 100644 --- a/api.go +++ b/api.go @@ -413,7 +413,7 @@ func (c *Client) SetS3TransferAccelerate(accelerateEndpoint string) { // - For signature v4 request if the connection is insecure compute only sha256. // - For signature v4 request if the connection is secure compute only md5. // - For anonymous request compute md5. -func (c *Client) hashMaterials() (hashAlgos map[string]hash.Hash, hashSums map[string][]byte) { +func (c *Client) hashMaterials(isMd5Requested bool) (hashAlgos map[string]hash.Hash, hashSums map[string][]byte) { hashSums = make(map[string][]byte) hashAlgos = make(map[string]hash.Hash) if c.overrideSignerType.IsV4() { @@ -427,6 +427,9 @@ func (c *Client) hashMaterials() (hashAlgos map[string]hash.Hash, hashSums map[s hashAlgos["md5"] = md5.New() } } + if isMd5Requested { + hashAlgos["md5"] = md5.New() + } return hashAlgos, hashSums } diff --git a/bucket-notification.go b/bucket-notification.go index 4714eadad..557e93c1c 100644 --- a/bucket-notification.go +++ b/bucket-notification.go @@ -81,7 +81,7 @@ func NewArn(partition, service, region, accountID, resource string) Arn { Resource: resource} } -// Return the string format of the ARN +// String returns the string format of the ARN func (arn Arn) String() string { return "arn:" + arn.Partition + ":" + arn.Service + ":" + arn.Region + ":" + arn.AccountID + ":" + arn.Resource } diff --git a/constants.go b/constants.go index 7f907c8b2..0fe9c9406 100644 --- a/constants.go +++ b/constants.go @@ -65,3 +65,6 @@ const amzWebsiteRedirectLocation = "X-Amz-Website-Redirect-Location" const amzTaggingHeader = "X-Amz-Tagging" const amzTaggingHeaderDirective = "X-Amz-Tagging-Directive" const amzTaggingHeaderCount = "X-Amz-Tagging-Count" + +// LegalHold Header constants +const amzLegalHoldHeader = "X-Amz-Object-Lock-Legal-Hold" diff --git a/docs/API.md b/docs/API.md index 0f80a0d70..4fb128675 100644 --- a/docs/API.md +++ b/docs/API.md @@ -617,6 +617,7 @@ __minio.PutObjectOptions__ | `opts.ServerSideEncryption` | _encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v6) | | `opts.StorageClass` | _string_ | Specify storage class for the object. Supported values for MinIO server are `REDUCED_REDUNDANCY` and `STANDARD` | | `opts.WebsiteRedirectLocation` | _string_ | Specify a redirect for the object, to another object in the same bucket or to a external URL. | +| `opts.SendContentMd5` | _bool_ | Specify if you'd like to send `content-md5` header with PutObject operation. Note that setting this flag will cause higher memory usage because of in-memory `md5sum` calculation. | __Example__ @@ -939,6 +940,16 @@ __Parameters__ | `object` | _string_ | Name of the destination object | | `destOpts` | _minio.DestInfoOptions_ | Pointer to struct that allows user to set optional custom metadata, user tags, and server side encryption parameters. | +__minio.DestInfoOptions__ + +|Field | Type | Description | +|:--- |:--- | :--- | +| `destOpts.Encryption` | _encrypt.ServerSide_ | Interface provided by encrypt package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v6). | +| `destOpts.UserMetadata` | _map[string]string_ | Map of user meta data to be set on destination object. | +| `destOpts.UserTags` | _map[string]string_ | Map of user object tags to be set on destination object. | +| `destOpts.ReplaceTags` | _bool_ | Replace object tags of the destination object. | +| `destOpts.LegalHold` | _*minio.LegalHoldStatus_ | LegalHold(En|Dis)abled. | + __Example__ ```go diff --git a/examples/s3/setbucketencryption.go b/examples/s3/setbucketencryption.go index 85f7a49b2..fda57c65d 100644 --- a/examples/s3/setbucketencryption.go +++ b/examples/s3/setbucketencryption.go @@ -43,7 +43,7 @@ func main() { // Set default encryption configuration on a bucket config := minio.ServerSideEncryptionConfiguration{Rules: []minio.Rule{ - minio.Rule{ + { Apply: minio.ApplyServerSideEncryptionByDefault{ SSEAlgorithm: "AES256", // KmsMasterKeyID: "my-masterkey", diff --git a/post-policy.go b/post-policy.go index 8b0bf092d..e0bc51ceb 100644 --- a/post-policy.go +++ b/post-policy.go @@ -258,7 +258,7 @@ func (p *PostPolicy) addNewPolicy(policyCond policyCondition) error { return nil } -// Stringer interface for printing policy in json formatted string. +// String implement the Stringer interface for printing policy in json formatted string. func (p PostPolicy) String() string { return string(p.marshalJSON()) } From b84ba84fdfe7bf330be3b22165c981edc4b933c4 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 4 Mar 2020 20:38:29 -0800 Subject: [PATCH 115/215] list: Check EncodingType in list resp before decoding object names (#1238) --- api-list.go | 28 +++++++++++++++++++--------- post-policy.go | 2 +- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/api-list.go b/api-list.go index c48201ce7..1bcb64f3d 100644 --- a/api-list.go +++ b/api-list.go @@ -317,14 +317,14 @@ func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken s } for i, obj := range listBucketResult.Contents { - listBucketResult.Contents[i].Key, err = url.QueryUnescape(obj.Key) + listBucketResult.Contents[i].Key, err = decodeS3Name(obj.Key, listBucketResult.EncodingType) if err != nil { return listBucketResult, err } } for i, obj := range listBucketResult.CommonPrefixes { - listBucketResult.CommonPrefixes[i].Prefix, err = url.QueryUnescape(obj.Prefix) + listBucketResult.CommonPrefixes[i].Prefix, err = decodeS3Name(obj.Prefix, listBucketResult.EncodingType) if err != nil { return listBucketResult, err } @@ -500,21 +500,21 @@ func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimit } for i, obj := range listBucketResult.Contents { - listBucketResult.Contents[i].Key, err = url.QueryUnescape(obj.Key) + listBucketResult.Contents[i].Key, err = decodeS3Name(obj.Key, listBucketResult.EncodingType) if err != nil { return listBucketResult, err } } for i, obj := range listBucketResult.CommonPrefixes { - listBucketResult.CommonPrefixes[i].Prefix, err = url.QueryUnescape(obj.Prefix) + listBucketResult.CommonPrefixes[i].Prefix, err = decodeS3Name(obj.Prefix, listBucketResult.EncodingType) if err != nil { return listBucketResult, err } } if listBucketResult.NextMarker != "" { - listBucketResult.NextMarker, err = url.QueryUnescape(listBucketResult.NextMarker) + listBucketResult.NextMarker, err = decodeS3Name(listBucketResult.NextMarker, listBucketResult.EncodingType) if err != nil { return listBucketResult, err } @@ -697,25 +697,25 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker, return listMultipartUploadsResult, err } - listMultipartUploadsResult.NextKeyMarker, err = url.QueryUnescape(listMultipartUploadsResult.NextKeyMarker) + listMultipartUploadsResult.NextKeyMarker, err = decodeS3Name(listMultipartUploadsResult.NextKeyMarker, listMultipartUploadsResult.EncodingType) if err != nil { return listMultipartUploadsResult, err } - listMultipartUploadsResult.NextUploadIDMarker, err = url.QueryUnescape(listMultipartUploadsResult.NextUploadIDMarker) + listMultipartUploadsResult.NextUploadIDMarker, err = decodeS3Name(listMultipartUploadsResult.NextUploadIDMarker, listMultipartUploadsResult.EncodingType) if err != nil { return listMultipartUploadsResult, err } for i, obj := range listMultipartUploadsResult.Uploads { - listMultipartUploadsResult.Uploads[i].Key, err = url.QueryUnescape(obj.Key) + listMultipartUploadsResult.Uploads[i].Key, err = decodeS3Name(obj.Key, listMultipartUploadsResult.EncodingType) if err != nil { return listMultipartUploadsResult, err } } for i, obj := range listMultipartUploadsResult.CommonPrefixes { - listMultipartUploadsResult.CommonPrefixes[i].Prefix, err = url.QueryUnescape(obj.Prefix) + listMultipartUploadsResult.CommonPrefixes[i].Prefix, err = decodeS3Name(obj.Prefix, listMultipartUploadsResult.EncodingType) if err != nil { return listMultipartUploadsResult, err } @@ -837,3 +837,13 @@ func (c Client) listObjectPartsQuery(bucketName, objectName, uploadID string, pa } return listObjectPartsResult, nil } + +// Decode an S3 object name according to the encoding type +func decodeS3Name(name, encodingType string) (string, error) { + switch encodingType { + case "url": + return url.QueryUnescape(name) + default: + return name, nil + } +} diff --git a/post-policy.go b/post-policy.go index e0bc51ceb..017916f06 100644 --- a/post-policy.go +++ b/post-policy.go @@ -258,7 +258,7 @@ func (p *PostPolicy) addNewPolicy(policyCond policyCondition) error { return nil } -// String implement the Stringer interface for printing policy in json formatted string. +// String function for printing policy in json formatted string. func (p PostPolicy) String() string { return string(p.marshalJSON()) } From 0e08e7578d9099b1c59f29630a7c38850e2ce5a9 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 6 Mar 2020 12:15:49 -0800 Subject: [PATCH 116/215] Add PutObjectOptions.PartSize docs (#1239) --- docs/API.md | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/docs/API.md b/docs/API.md index 4fb128675..b49244e48 100644 --- a/docs/API.md +++ b/docs/API.md @@ -602,22 +602,24 @@ __Parameters__ __minio.PutObjectOptions__ -|Field | Type | Description | -|:--- |:--- | :--- | -| `opts.UserMetadata` | _map[string]string_ | Map of user metadata| -| `opts.UserTags` | _map[string]string_ | Map of user object tags | -| `opts.Progress` | _io.Reader_ | Reader to fetch progress of an upload | -| `opts.ContentType` | _string_ | Content type of object, e.g "application/text" | -| `opts.ContentEncoding` | _string_ | Content encoding of object, e.g "gzip" | -| `opts.ContentDisposition` | _string_ | Content disposition of object, "inline" | -| `opts.ContentLanguage` | _string_ | Content language of object, e.g "French" | -| `opts.CacheControl` | _string_ | Used to specify directives for caching mechanisms in both requests and responses e.g "max-age=600"| -| `opts.Mode` | _*minio.RetentionMode_ | Retention mode to be set, e.g "COMPLIANCE" | -| `opts.RetainUntilDate` | _*time.Time_ | Time until which the retention applied is valid| -| `opts.ServerSideEncryption` | _encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v6) | -| `opts.StorageClass` | _string_ | Specify storage class for the object. Supported values for MinIO server are `REDUCED_REDUNDANCY` and `STANDARD` | -| `opts.WebsiteRedirectLocation` | _string_ | Specify a redirect for the object, to another object in the same bucket or to a external URL. | -| `opts.SendContentMd5` | _bool_ | Specify if you'd like to send `content-md5` header with PutObject operation. Note that setting this flag will cause higher memory usage because of in-memory `md5sum` calculation. | +| Field | Type | Description | +|:-------------------------------|:-----------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `opts.UserMetadata` | _map[string]string_ | Map of user metadata | +| `opts.UserTags` | _map[string]string_ | Map of user object tags | +| `opts.Progress` | _io.Reader_ | Reader to fetch progress of an upload | +| `opts.ContentType` | _string_ | Content type of object, e.g "application/text" | +| `opts.ContentEncoding` | _string_ | Content encoding of object, e.g "gzip" | +| `opts.ContentDisposition` | _string_ | Content disposition of object, "inline" | +| `opts.ContentLanguage` | _string_ | Content language of object, e.g "French" | +| `opts.CacheControl` | _string_ | Used to specify directives for caching mechanisms in both requests and responses e.g "max-age=600" | +| `opts.Mode` | _*minio.RetentionMode_ | Retention mode to be set, e.g "COMPLIANCE" | +| `opts.RetainUntilDate` | _*time.Time_ | Time until which the retention applied is valid | +| `opts.ServerSideEncryption` | _encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v6) | +| `opts.StorageClass` | _string_ | Specify storage class for the object. Supported values for MinIO server are `REDUCED_REDUNDANCY` and `STANDARD` | +| `opts.WebsiteRedirectLocation` | _string_ | Specify a redirect for the object, to another object in the same bucket or to a external URL. | +| `opts.SendContentMd5` | _bool_ | Specify if you'd like to send `content-md5` header with PutObject operation. Note that setting this flag will cause higher memory usage because of in-memory `md5sum` calculation. | +| `opts.PartSize` | _uint64_ | Specify a custom part size used for uploading the object | + __Example__ From b882ba63d570968489accead9c3c8141ea09c222 Mon Sep 17 00:00:00 2001 From: poornas Date: Fri, 6 Mar 2020 15:11:01 -0800 Subject: [PATCH 117/215] Add (Get/Set)ObjectLockConfig API functions (#1240) GetObjectLockConfig will report object lock enabled status in addition to the object lock config. --- api-object-lock.go | 17 +++++++++++++---- docs/API.md | 19 ++++++++++--------- ...ctlockconfig.go => getobjectlockconfig.go} | 6 ++++-- ...ctlockconfig.go => setobjectlockconfig.go} | 2 +- 4 files changed, 28 insertions(+), 16 deletions(-) rename examples/s3/{getbucketobjectlockconfig.go => getobjectlockconfig.go} (85%) rename examples/s3/{setbucketobjectlockconfig.go => setobjectlockconfig.go} (94%) diff --git a/api-object-lock.go b/api-object-lock.go index fa8b81709..14168ef39 100644 --- a/api-object-lock.go +++ b/api-object-lock.go @@ -183,8 +183,8 @@ func (c Client) SetBucketObjectLockConfig(bucketName string, mode *RetentionMode return nil } -// GetBucketObjectLockConfig gets object lock configuration of given bucket. -func (c Client) GetBucketObjectLockConfig(bucketName string) (objectLock string, mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { +// GetObjectLockConfig gets object lock configuration of given bucket. +func (c Client) GetObjectLockConfig(bucketName string) (objectLock string, mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return "", nil, nil, nil, err @@ -224,9 +224,18 @@ func (c Client) GetBucketObjectLockConfig(bucketName string) (objectLock string, years := Years unit = &years } - return config.ObjectLockEnabled, mode, validity, unit, nil } + return config.ObjectLockEnabled, nil, nil, nil, nil +} + +// GetBucketObjectLockConfig gets object lock configuration of given bucket. +func (c Client) GetBucketObjectLockConfig(bucketName string) (mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { + _, mode, validity, unit, err = c.GetObjectLockConfig(bucketName) + return mode, validity, unit, err +} - return "", nil, nil, nil, nil +// SetObjectLockConfig sets object lock configuration in given bucket. mode, validity and unit are either all set or all nil. +func (c Client) SetObjectLockConfig(bucketName string, mode *RetentionMode, validity *uint, unit *ValidityUnit) error { + return c.SetBucketObjectLockConfig(bucketName, mode, validity, unit) } diff --git a/docs/API.md b/docs/API.md index b49244e48..cbc6d53da 100644 --- a/docs/API.md +++ b/docs/API.md @@ -60,8 +60,8 @@ func main() { | [`ListObjects`](#ListObjects) | [`RemoveObjects`](#RemoveObjects) | | | [`ListenBucketNotification`](#ListenBucketNotification) | | | [`ListObjectsV2`](#ListObjectsV2) | [`RemoveIncompleteUpload`](#RemoveIncompleteUpload) | | | [`SetBucketLifecycle`](#SetBucketLifecycle) | | | [`ListIncompleteUploads`](#ListIncompleteUploads) | [`FPutObject`](#FPutObject) | [`FPutObject`](#FPutObject) | | [`GetBucketLifecycle`](#GetBucketLifecycle) | | -| | [`FGetObject`](#FGetObject) | [`FGetObject`](#FGetObject) | | [`SetBucketObjectLockConfig`](#SetBucketObjectLockConfig) | | -| | [`ComposeObject`](#ComposeObject) | [`ComposeObject`](#ComposeObject) | | [`GetBucketObjectLockConfig`](#GetBucketObjectLockConfig) | | +| | [`FGetObject`](#FGetObject) | [`FGetObject`](#FGetObject) | | [`SetObjectLockConfig`](#SetObjectLockConfig) | | +| | [`ComposeObject`](#ComposeObject) | [`ComposeObject`](#ComposeObject) | | [`GetObjectLockConfig`](#GetObjectLockConfig) | | | | [`NewSourceInfo`](#NewSourceInfo) | [`NewSourceInfo`](#NewSourceInfo) | | [`EnableVersioning`](#EnableVersioning) | | | | [`NewDestinationInfo`](#NewDestinationInfo) | [`NewDestinationInfo`](#NewDestinationInfo) | | [`DisableVersioning`](#DisableVersioning) | | | | [`PutObjectWithContext`](#PutObjectWithContext) | [`PutObjectWithContext`](#PutObjectWithContext) | | [`SetBucketEncryption`](#SetBucketEncryption) | @@ -2089,8 +2089,8 @@ if err != nil { // "my-bucket" is successfully deleted/removed. ``` - -### SetBucketObjectLockConfig(bucketname, mode *RetentionMode, validity *uint, unit *ValidityUnit) error + +### SetObjectLockConfig(bucketname, mode *RetentionMode, validity *uint, unit *ValidityUnit) error Set object lock configuration in given bucket. mode, validity and unit are either all set or all nil. __Parameters__ @@ -2115,15 +2115,15 @@ mode := Governance validity := uint(30) unit := Days -err = minioClient.SetBucketObjectLockConfig("my-bucketname", &mode, &validity, &unit) +err = minioClient.SetObjectLockConfig("my-bucketname", &mode, &validity, &unit) if err != nil { fmt.Println(err) return } ``` - -### GetBucketObjectLockConfig(bucketName) (*RetentionMode, *uint, *ValidityUnit, error) + +### GetObjectLockConfig(bucketName) (objectLock,*RetentionMode, *uint, *ValidityUnit, error) Get object lock configuration of given bucket. __Parameters__ @@ -2138,6 +2138,7 @@ __Return Values__ |Param |Type |Description | |:---|:---| :---| +|`objectLock` | _objectLock_ |lock enabled status | |`mode` | _RetentionMode_ |Current retention mode | |`validity` | _uint_ |Current validity period | |`unit` | _ValidityUnit_ |Unit of validity period | @@ -2146,11 +2147,11 @@ __Return Values__ __Example__ ```go -mode, validity, unit, err := minioClient.GetObjectLockConfig("my-bucketname") +enabled, mode, validity, unit, err := minioClient.GetObjectLockConfig("my-bucketname") if err != nil { log.Fatalln(err) } - +fmt.Println("object lock is %s for this bucket",enabled) if mode != nil { fmt.Printf("%v mode is enabled for %v %v for bucket 'my-bucketname'\n", *mode, *validity, *unit) } else { diff --git a/examples/s3/getbucketobjectlockconfig.go b/examples/s3/getobjectlockconfig.go similarity index 85% rename from examples/s3/getbucketobjectlockconfig.go rename to examples/s3/getobjectlockconfig.go index ddd9f9250..cfdced1a7 100644 --- a/examples/s3/getbucketobjectlockconfig.go +++ b/examples/s3/getobjectlockconfig.go @@ -22,6 +22,7 @@ package main import ( "fmt" "log" + "os" minio "github.com/minio/minio-go/v6" ) @@ -35,7 +36,7 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) if err != nil { log.Fatalln(err) } @@ -43,10 +44,11 @@ func main() { // s3Client.TraceOn(os.Stderr) // Get object lock configuration. - mode, validity, unit, err := s3Client.GetBucketObjectLockConfig("my-bucketname") + enabled, mode, validity, unit, err := s3Client.GetObjectLockConfig("tbucket13a") if err != nil { log.Fatalln(err) } + fmt.Printf("object lock is %v for bucket 'my-bucketname'\n", enabled) if mode != nil { fmt.Printf("%v mode is enabled for %v %v for bucket 'my-bucketname'\n", *mode, *validity, *unit) diff --git a/examples/s3/setbucketobjectlockconfig.go b/examples/s3/setobjectlockconfig.go similarity index 94% rename from examples/s3/setbucketobjectlockconfig.go rename to examples/s3/setobjectlockconfig.go index 8034969a5..6169f1770 100644 --- a/examples/s3/setbucketobjectlockconfig.go +++ b/examples/s3/setobjectlockconfig.go @@ -46,7 +46,7 @@ func main() { validity := uint(30) unit := minio.Days - err = s3Client.SetBucketObjectLockConfig("my-bucketname", &mode, &validity, &unit) + err = s3Client.SetObjectLockConfig("my-bucketname", &mode, &validity, &unit) if err != nil { log.Fatalln(err) } From 925dba936c72d0fd69b04255bb98fa1d0ee4fef7 Mon Sep 17 00:00:00 2001 From: Chris Torek Date: Mon, 9 Mar 2020 22:16:45 -0700 Subject: [PATCH 118/215] fix: close and remove .minio.part file on errors (#1243) If a .minio.part file becomes damaged, it lingers in the file system and can cause future attempts to get the object to fail. Each failure leaves a file open. To fix this, if we are returning early from fGetObjectWithContext (which always returns an error) and we haven't closed the file safely yet, close and remove it now. --- api-get-object-file.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/api-get-object-file.go b/api-get-object-file.go index 0684b2b3a..c2d277514 100644 --- a/api-get-object-file.go +++ b/api-get-object-file.go @@ -87,6 +87,17 @@ func (c Client) fGetObjectWithContext(ctx context.Context, bucketName, objectNam return err } + // If we return early with an error, be sure to close and delete + // filePart. If we have an error along the way there is a chance + // that filePart is somehow damaged, and we should discard it. + closeAndRemove := true + defer func() { + if closeAndRemove { + _ = filePart.Close() + _ = os.Remove(filePartPath) + } + }() + // Issue Stat to get the current offset. st, err = filePart.Stat() if err != nil { @@ -111,6 +122,7 @@ func (c Client) fGetObjectWithContext(ctx context.Context, bucketName, objectNam } // Close the file before rename, this is specifically needed for Windows users. + closeAndRemove = false if err = filePart.Close(); err != nil { return err } From 71b53217fa2cfdd5e166d3ffe43b8bd2739715df Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Sun, 15 Mar 2020 21:13:40 +0000 Subject: [PATCH 119/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 1a4e0699b..0c3b40833 100644 --- a/api.go +++ b/api.go @@ -104,7 +104,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.50" + libraryVersion = "v6.0.51" ) // User Agent should always following the below style. From 93b7236b8909552a063213665841517eb39ae9eb Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 16 Mar 2020 13:44:08 -0700 Subject: [PATCH 120/215] simplify extracting metadata in Head/Get object (#1245) ensure that we only look for correct headers needed, rather than filtering out headers which may be many, this will ensure that we shall never copy unexpected headers. --- api-get-object.go | 36 ++----------- api-put-object.go | 39 +++++++++------ api-stat.go | 109 +--------------------------------------- constants.go | 21 ++++---- utils.go | 125 +++++++++++++++++++++++++++++++++++++++------- utils_test.go | 16 ------ 6 files changed, 146 insertions(+), 200 deletions(-) diff --git a/api-get-object.go b/api-get-object.go index c73f4a6b4..0c1653cdc 100644 --- a/api-get-object.go +++ b/api-get-object.go @@ -23,9 +23,7 @@ import ( "fmt" "io" "net/http" - "strings" "sync" - "time" "github.com/minio/minio-go/v6/pkg/s3utils" ) @@ -635,38 +633,10 @@ func (c Client) getObject(ctx context.Context, bucketName, objectName string, op } } - // Trim off the odd double quotes from ETag in the beginning and end. - md5sum := trimEtag(resp.Header.Get("ETag")) - - // Parse the date. - date, err := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified")) + objectStat, err := ToObjectInfo(bucketName, objectName, resp.Header) if err != nil { - msg := "Last-Modified time format not recognized. " + reportIssue - return nil, ObjectInfo{}, nil, ErrorResponse{ - Code: "InternalError", - Message: msg, - RequestID: resp.Header.Get("x-amz-request-id"), - HostID: resp.Header.Get("x-amz-id-2"), - Region: resp.Header.Get("x-amz-bucket-region"), - } - } - - // Get content-type. - contentType := strings.TrimSpace(resp.Header.Get("Content-Type")) - if contentType == "" { - contentType = "application/octet-stream" - } - - objectStat := ObjectInfo{ - ETag: md5sum, - Key: objectName, - Size: resp.ContentLength, - LastModified: date, - ContentType: contentType, - // Extract only the relevant header keys describing the object. - // following function filters out a list of standard set of keys - // which are not part of object metadata. - Metadata: extractObjMetadata(resp.Header), + closeResponse(resp) + return nil, objectStat, resp.Header, nil } // do not close body here, caller will close diff --git a/api-put-object.go b/api-put-object.go index 4598f58ce..0147a060b 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -71,51 +71,58 @@ func (opts PutObjectOptions) getNumThreads() (numThreads int) { func (opts PutObjectOptions) Header() (header http.Header) { header = make(http.Header) - if opts.ContentType != "" { - header["Content-Type"] = []string{opts.ContentType} - } else { - header["Content-Type"] = []string{"application/octet-stream"} + contentType := opts.ContentType + if contentType == "" { + contentType = "application/octet-stream" } + header.Set("Content-Type", contentType) + if opts.ContentEncoding != "" { - header["Content-Encoding"] = []string{opts.ContentEncoding} + header.Set("Content-Encoding", opts.ContentEncoding) } if opts.ContentDisposition != "" { - header["Content-Disposition"] = []string{opts.ContentDisposition} + header.Set("Content-Disposition", opts.ContentDisposition) } if opts.ContentLanguage != "" { - header["Content-Language"] = []string{opts.ContentLanguage} + header.Set("Content-Language", opts.ContentLanguage) } if opts.CacheControl != "" { - header["Cache-Control"] = []string{opts.CacheControl} + header.Set("Cache-Control", opts.CacheControl) } if opts.Mode != nil { - header["x-amz-object-lock-mode"] = []string{opts.Mode.String()} + header.Set("X-Amz-Object-Lock-Mode", opts.Mode.String()) } + if opts.RetainUntilDate != nil { - header["x-amz-object-lock-retain-until-date"] = []string{opts.RetainUntilDate.Format(time.RFC3339)} + header.Set("X-Amz-Object-Lock-Retain-Until-Date", opts.RetainUntilDate.Format(time.RFC3339)) } if opts.LegalHold != "" { - header[amzLegalHoldHeader] = []string{opts.LegalHold.String()} + header.Set(amzLegalHoldHeader, opts.LegalHold.String()) } + if opts.ServerSideEncryption != nil { opts.ServerSideEncryption.Marshal(header) } + if opts.StorageClass != "" { - header[amzStorageClass] = []string{opts.StorageClass} + header.Set(amzStorageClass, opts.StorageClass) } + if opts.WebsiteRedirectLocation != "" { - header[amzWebsiteRedirectLocation] = []string{opts.WebsiteRedirectLocation} + header.Set(amzWebsiteRedirectLocation, opts.WebsiteRedirectLocation) } + if len(opts.UserTags) != 0 { - header[amzTaggingHeader] = []string{s3utils.TagEncode(opts.UserTags)} + header.Set(amzTaggingHeader, s3utils.TagEncode(opts.UserTags)) } + for k, v := range opts.UserMetadata { if !isAmzHeader(k) && !isStandardHeader(k) && !isStorageClassHeader(k) { - header["X-Amz-Meta-"+k] = []string{v} + header.Set("X-Amz-Meta-"+k, v) } else { - header[k] = []string{v} + header.Set(k, v) } } return diff --git a/api-stat.go b/api-stat.go index 45844c9cb..5da981b6c 100644 --- a/api-stat.go +++ b/api-stat.go @@ -20,9 +20,6 @@ package minio import ( "context" "net/http" - "strconv" - "strings" - "time" "github.com/minio/minio-go/v6/pkg/s3utils" ) @@ -64,39 +61,6 @@ func (c Client) BucketExistsWithContext(ctx context.Context, bucketName string) return true, nil } -// List of header keys to be filtered, usually -// from all S3 API http responses. -var defaultFilterKeys = []string{ - "Connection", - "Transfer-Encoding", - "Accept-Ranges", - "Date", - "Server", - "Vary", - "x-amz-bucket-region", - "x-amz-request-id", - "x-amz-id-2", - "Content-Security-Policy", - "X-Xss-Protection", - amzTaggingHeaderCount, - // Add new headers to be ignored. -} - -// Extract only necessary metadata header key/values by -// filtering them out with a list of custom header keys. -func extractObjMetadata(header http.Header) http.Header { - filterKeys := append([]string{ - "ETag", - "Content-Length", - "Last-Modified", - "Content-Type", - "Expires", - "X-Cache", - "X-Cache-Lookup", - }, defaultFilterKeys...) - return filterHeader(header, filterKeys) -} - // StatObject verifies if object exists and you have permission to access. func (c Client) StatObject(bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { return c.StatObjectWithContext(context.Background(), bucketName, objectName, opts) @@ -142,76 +106,5 @@ func (c Client) statObject(ctx context.Context, bucketName, objectName string, o } } - // Trim off the odd double quotes from ETag in the beginning and end. - md5sum := trimEtag(resp.Header.Get("ETag")) - - // Parse content length is exists - var size int64 = -1 - contentLengthStr := resp.Header.Get("Content-Length") - if contentLengthStr != "" { - size, err = strconv.ParseInt(contentLengthStr, 10, 64) - if err != nil { - // Content-Length is not valid - return ObjectInfo{}, ErrorResponse{ - Code: "InternalError", - Message: "Content-Length is invalid. " + reportIssue, - BucketName: bucketName, - Key: objectName, - RequestID: resp.Header.Get("x-amz-request-id"), - HostID: resp.Header.Get("x-amz-id-2"), - Region: resp.Header.Get("x-amz-bucket-region"), - } - } - } - - // Parse Last-Modified has http time format. - date, err := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified")) - if err != nil { - return ObjectInfo{}, ErrorResponse{ - Code: "InternalError", - Message: "Last-Modified time format is invalid. " + reportIssue, - BucketName: bucketName, - Key: objectName, - RequestID: resp.Header.Get("x-amz-request-id"), - HostID: resp.Header.Get("x-amz-id-2"), - Region: resp.Header.Get("x-amz-bucket-region"), - } - } - - // Fetch content type if any present. - contentType := strings.TrimSpace(resp.Header.Get("Content-Type")) - if contentType == "" { - contentType = "application/octet-stream" - } - - expiryStr := resp.Header.Get("Expires") - var expTime time.Time - if t, err := time.Parse(http.TimeFormat, expiryStr); err == nil { - expTime = t.UTC() - } - - metadata := extractObjMetadata(resp.Header) - userMetadata := map[string]string{} - const xamzmeta = "x-amz-meta-" - const xamzmetaLen = len(xamzmeta) - for k, v := range metadata { - if strings.HasPrefix(strings.ToLower(k), xamzmeta) { - userMetadata[k[xamzmetaLen:]] = v[0] - } - } - - // Save object metadata info. - return ObjectInfo{ - ETag: md5sum, - Key: objectName, - Size: size, - LastModified: date, - ContentType: contentType, - Expires: expTime, - // Extract only the relevant header keys describing the object. - // following function filters out a list of standard set of keys - // which are not part of object metadata. - Metadata: metadata, - UserMetadata: userMetadata, - }, nil + return ToObjectInfo(bucketName, objectName, resp.Header) } diff --git a/constants.go b/constants.go index 0fe9c9406..9d0166250 100644 --- a/constants.go +++ b/constants.go @@ -55,16 +55,17 @@ const ( iso8601DateFormat = "20060102T150405Z" ) -// Storage class header constant. -const amzStorageClass = "X-Amz-Storage-Class" +const ( + // Storage class header. + amzStorageClass = "X-Amz-Storage-Class" -// Website redirect location header constant -const amzWebsiteRedirectLocation = "X-Amz-Website-Redirect-Location" + // Website redirect location header + amzWebsiteRedirectLocation = "X-Amz-Website-Redirect-Location" -// Tagging header constants -const amzTaggingHeader = "X-Amz-Tagging" -const amzTaggingHeaderDirective = "X-Amz-Tagging-Directive" -const amzTaggingHeaderCount = "X-Amz-Tagging-Count" + // Object Tagging headers + amzTaggingHeader = "X-Amz-Tagging" + amzTaggingHeaderDirective = "X-Amz-Tagging-Directive" -// LegalHold Header constants -const amzLegalHoldHeader = "X-Amz-Object-Lock-Legal-Hold" + // Object legal hold header + amzLegalHoldHeader = "X-Amz-Object-Lock-Legal-Hold" +) diff --git a/utils.go b/utils.go index 5b324547d..a6ffedcba 100644 --- a/utils.go +++ b/utils.go @@ -28,6 +28,7 @@ import ( "net/http" "net/url" "regexp" + "strconv" "strings" "time" @@ -159,27 +160,117 @@ func isValidExpiry(expires time.Duration) error { return nil } -// make a copy of http.Header -func cloneHeader(h http.Header) http.Header { - h2 := make(http.Header, len(h)) - for k, vv := range h { - vv2 := make([]string, len(vv)) - copy(vv2, vv) - h2[k] = vv2 +// Extract only necessary metadata header key/values by +// filtering them out with a list of custom header keys. +func extractObjMetadata(header http.Header) http.Header { + preserveKeys := []string{ + "Content-Type", + "Cache-Control", + "Content-Encoding", + "Content-Language", + "Content-Disposition", + "X-Amz-Storage-Class", + "X-Amz-Object-Lock-Mode", + "X-Amz-Object-Lock-Retain-Until-Date", + "X-Amz-Object-Lock-Legal-Hold", + "X-Amz-Website-Redirect-Location", + "X-Amz-Meta-", + // Add new headers to be preserved. + // if you add new headers here, please extend + // PutObjectOptions{} to preserve them + // upon upload as well. } - return h2 + filteredHeader := make(http.Header) + for k, v := range header { + var found bool + for _, prefix := range preserveKeys { + if !strings.HasPrefix(k, prefix) { + continue + } + found = true + break + } + if found { + filteredHeader[k] = v + } + } + return filteredHeader } -// Filter relevant response headers from -// the HEAD, GET http response. The function takes -// a list of headers which are filtered out and -// returned as a new http header. -func filterHeader(header http.Header, filterKeys []string) (filteredHeader http.Header) { - filteredHeader = cloneHeader(header) - for _, key := range filterKeys { - filteredHeader.Del(key) +// ToObjectInfo converts http header values into ObjectInfo type, +// extracts metadata and fills in all the necessary fields in ObjectInfo. +func ToObjectInfo(bucketName string, objectName string, h http.Header) (ObjectInfo, error) { + var err error + // Trim off the odd double quotes from ETag in the beginning and end. + etag := trimEtag(h.Get("ETag")) + + // Parse content length is exists + var size int64 = -1 + contentLengthStr := h.Get("Content-Length") + if contentLengthStr != "" { + size, err = strconv.ParseInt(contentLengthStr, 10, 64) + if err != nil { + // Content-Length is not valid + return ObjectInfo{}, ErrorResponse{ + Code: "InternalError", + Message: "Content-Length is invalid. " + reportIssue, + BucketName: bucketName, + Key: objectName, + RequestID: h.Get("x-amz-request-id"), + HostID: h.Get("x-amz-id-2"), + Region: h.Get("x-amz-bucket-region"), + } + } } - return filteredHeader + + // Parse Last-Modified has http time format. + date, err := time.Parse(http.TimeFormat, h.Get("Last-Modified")) + if err != nil { + return ObjectInfo{}, ErrorResponse{ + Code: "InternalError", + Message: "Last-Modified time format is invalid. " + reportIssue, + BucketName: bucketName, + Key: objectName, + RequestID: h.Get("x-amz-request-id"), + HostID: h.Get("x-amz-id-2"), + Region: h.Get("x-amz-bucket-region"), + } + } + + // Fetch content type if any present. + contentType := strings.TrimSpace(h.Get("Content-Type")) + if contentType == "" { + contentType = "application/octet-stream" + } + + expiryStr := h.Get("Expires") + var expTime time.Time + if t, err := time.Parse(http.TimeFormat, expiryStr); err == nil { + expTime = t.UTC() + } + + metadata := extractObjMetadata(h) + userMetadata := make(map[string]string) + for k, v := range metadata { + if strings.HasPrefix(k, "X-Amz-Meta-") { + userMetadata[strings.TrimPrefix(k, "X-Amz-Meta-")] = v[0] + } + } + + // Save object metadata info. + return ObjectInfo{ + ETag: etag, + Key: objectName, + Size: size, + LastModified: date, + ContentType: contentType, + Expires: expTime, + // Extract only the relevant header keys describing the object. + // following function filters out a list of standard set of keys + // which are not part of object metadata. + Metadata: metadata, + UserMetadata: userMetadata, + }, nil } // regCred matches credential string in HTTP header diff --git a/utils_test.go b/utils_test.go index 03a0a82d5..70f80fa91 100644 --- a/utils_test.go +++ b/utils_test.go @@ -19,7 +19,6 @@ package minio import ( "fmt" - "net/http" "net/url" "testing" "time" @@ -52,21 +51,6 @@ func TestRedactSignature(t *testing.T) { } } -// Tests filter header function by filtering out -// some custom header keys. -func TestFilterHeader(t *testing.T) { - header := http.Header{} - header.Set("Content-Type", "binary/octet-stream") - header.Set("Content-Encoding", "gzip") - newHeader := filterHeader(header, []string{"Content-Type"}) - if len(newHeader) > 1 { - t.Fatalf("Unexpected size of the returned header, should be 1, got %d", len(newHeader)) - } - if newHeader.Get("Content-Encoding") != "gzip" { - t.Fatalf("Unexpected content-encoding value, expected 'gzip', got %s", newHeader.Get("Content-Encoding")) - } -} - // Tests for 'getEndpointURL(endpoint string, inSecure bool)'. func TestGetEndpointURL(t *testing.T) { testCases := []struct { From a9505dfe1267fa6961f4f80367269724724d81d2 Mon Sep 17 00:00:00 2001 From: BigUstad Date: Wed, 18 Mar 2020 07:37:31 -0700 Subject: [PATCH 121/215] Check for correct http status in remove object tagging (#1248) --- api-object-tagging.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api-object-tagging.go b/api-object-tagging.go index fdf8fab91..66568063c 100644 --- a/api-object-tagging.go +++ b/api-object-tagging.go @@ -155,7 +155,8 @@ func (c Client) RemoveObjectTaggingWithContext(ctx context.Context, bucketName, } if resp != nil { - if resp.StatusCode != http.StatusOK { + // S3 returns "204 No content" after Object tag deletion. + if resp.StatusCode != http.StatusNoContent { return httpRespToErrorResponse(resp, bucketName, objectName) } } From d19a75581dbdb15baa1793425e4c7aef316bb1e9 Mon Sep 17 00:00:00 2001 From: Fedor Ortyanov Date: Thu, 19 Mar 2020 19:15:35 +0300 Subject: [PATCH 122/215] fix: Context cancellation not handled (#1250) --- api.go | 8 ++++---- retry.go | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/api.go b/api.go index 0c3b40833..66b06243d 100644 --- a/api.go +++ b/api.go @@ -591,16 +591,16 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque } } - // Create a done channel to control 'newRetryTimer' go routine. - doneCh := make(chan struct{}, 1) + // Create cancel context to control 'newRetryTimer' go routine. + retryCtx, cancel := context.WithCancel(ctx) // Indicate to our routine to exit cleanly upon return. - defer close(doneCh) + defer cancel() // Blank indentifier is kept here on purpose since 'range' without // blank identifiers is only supported since go1.4 // https://golang.org/doc/go1.4#forrange. - for range c.newRetryTimer(reqRetry, DefaultRetryUnit, DefaultRetryCap, MaxJitter, doneCh) { + for range c.newRetryTimer(retryCtx, reqRetry, DefaultRetryUnit, DefaultRetryCap, MaxJitter) { // Retry executes the following function body if request has an // error until maxRetries have been exhausted, retry attempts are // performed after waiting for a given period of time in a diff --git a/retry.go b/retry.go index f9c1095c0..f85a3e76a 100644 --- a/retry.go +++ b/retry.go @@ -18,6 +18,7 @@ package minio import ( + "context" "net/http" "time" ) @@ -41,7 +42,7 @@ const DefaultRetryCap = time.Second * 30 // newRetryTimer creates a timer with exponentially increasing // delays until the maximum retry attempts are reached. -func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duration, jitter float64, doneCh chan struct{}) <-chan int { +func (c Client) newRetryTimer(ctx context.Context, maxRetry int, unit time.Duration, cap time.Duration, jitter float64) <-chan int { attemptCh := make(chan int) // computes the exponential backoff duration according to @@ -69,14 +70,13 @@ func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duratio go func() { defer close(attemptCh) for i := 0; i < maxRetry; i++ { + attemptCh <- i + 1 + select { - // Attempts start from 1. - case attemptCh <- i + 1: - case <-doneCh: - // Stop the routine. + case <-time.After(exponentialBackoffWait(i)): + case <-ctx.Done(): return } - time.Sleep(exponentialBackoffWait(i)) } }() return attemptCh From 097caa7760c782f9f7c238eebf593aa404b2ef35 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 19 Mar 2020 12:21:31 -0700 Subject: [PATCH 123/215] extract userTags from Get/Head request (#1249) --- api-datatypes.go | 3 ++ api-get-object.go | 2 +- pkg/s3utils/utils.go | 48 ++++++++++++++---------- pkg/s3utils/utils_test.go | 77 +++++++++++++++++++++++++++++++++++++++ transport.go | 3 +- utils.go | 3 ++ 6 files changed, 115 insertions(+), 21 deletions(-) diff --git a/api-datatypes.go b/api-datatypes.go index e81a3bd41..58e3c5e0e 100644 --- a/api-datatypes.go +++ b/api-datatypes.go @@ -82,6 +82,9 @@ type ObjectInfo struct { // x-amz-meta-* headers stripped "x-amz-meta-" prefix containing the first value. UserMetadata StringMap `json:"userMetadata"` + // x-amz-tagging values in their k/v values. + UserTags map[string]string `json:"userTags"` + // Owner name. Owner struct { DisplayName string `json:"name"` diff --git a/api-get-object.go b/api-get-object.go index 0c1653cdc..14370b0cd 100644 --- a/api-get-object.go +++ b/api-get-object.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/s3utils/utils.go b/pkg/s3utils/utils.go index 982dddf4b..d01878dc1 100644 --- a/pkg/s3utils/utils.go +++ b/pkg/s3utils/utils.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -229,29 +229,39 @@ func QueryEncode(v url.Values) string { return buf.String() } +// TagDecode - decodes canonical tag into map of key and value. +func TagDecode(ctag string) map[string]string { + if ctag == "" { + return map[string]string{} + } + tags := strings.Split(ctag, "&") + tagMap := make(map[string]string, len(tags)) + var err error + for _, tag := range tags { + kvs := strings.SplitN(tag, "=", 2) + if len(kvs) == 0 { + return map[string]string{} + } + if len(kvs) == 1 { + return map[string]string{} + } + tagMap[kvs[0]], err = url.PathUnescape(kvs[1]) + if err != nil { + continue + } + } + return tagMap +} + // TagEncode - encodes tag values in their URL encoded form. In // addition to the percent encoding performed by urlEncodePath() used // here, it also percent encodes '/' (forward slash) func TagEncode(tags map[string]string) string { - if tags == nil { - return "" + values := url.Values{} + for k, v := range tags { + values[k] = []string{v} } - var buf bytes.Buffer - keys := make([]string, 0, len(tags)) - for k := range tags { - keys = append(keys, k) - } - sort.Strings(keys) - for _, k := range keys { - v := tags[k] - prefix := percentEncodeSlash(EncodePath(k)) + "=" - if buf.Len() > 0 { - buf.WriteByte('&') - } - buf.WriteString(prefix) - buf.WriteString(percentEncodeSlash(EncodePath(v))) - } - return buf.String() + return QueryEncode(values) } // if object matches reserved string, no need to encode them diff --git a/pkg/s3utils/utils_test.go b/pkg/s3utils/utils_test.go index e2fd3089d..125456031 100644 --- a/pkg/s3utils/utils_test.go +++ b/pkg/s3utils/utils_test.go @@ -20,6 +20,7 @@ package s3utils import ( "errors" "net/url" + "reflect" "testing" ) @@ -312,6 +313,81 @@ func TestQueryEncode(t *testing.T) { } } +// Tests tag decode to map +func TestTagDecode(t *testing.T) { + testCases := []struct { + // canonical input + canonicalInput string + + // Expected result. + resultMap map[string]string + }{ + {"k=thisisthe%25url", map[string]string{"k": "thisisthe%url"}}, + {"k=%E6%9C%AC%E8%AA%9E", map[string]string{"k": "本語"}}, + {"k=%E6%9C%AC%E8%AA%9E.1", map[string]string{"k": "本語.1"}}, + {"k=%3E123", map[string]string{"k": ">123"}}, + {"k=myurl%23link", map[string]string{"k": "myurl#link"}}, + {"k=space%20in%20url", map[string]string{"k": "space in url"}}, + {"k=url%2Bpath", map[string]string{"k": "url+path"}}, + {"k=url%2Fpath", map[string]string{"k": "url/path"}}, + } + + for _, testCase := range testCases { + testCase := testCase + t.Run("", func(t *testing.T) { + gotResult := TagDecode(testCase.canonicalInput) + if !reflect.DeepEqual(testCase.resultMap, gotResult) { + t.Errorf("Expected %s, got %s", testCase.resultMap, gotResult) + } + }) + } +} + +// Tests tag encode function for user tags. +func TestTagEncode(t *testing.T) { + testCases := []struct { + // Input. + inputMap map[string]string + // Expected result. + result string + }{ + {map[string]string{ + "k": "thisisthe%url", + }, "k=thisisthe%25url"}, + {map[string]string{ + "k": "本語", + }, "k=%E6%9C%AC%E8%AA%9E"}, + {map[string]string{ + "k": "本語.1", + }, "k=%E6%9C%AC%E8%AA%9E.1"}, + {map[string]string{ + "k": ">123", + }, "k=%3E123"}, + {map[string]string{ + "k": "myurl#link", + }, "k=myurl%23link"}, + {map[string]string{ + "k": "space in url", + }, "k=space%20in%20url"}, + {map[string]string{ + "k": "url+path", + }, "k=url%2Bpath"}, + {map[string]string{ + "k": "url/path", + }, "k=url%2Fpath"}, + } + + for _, testCase := range testCases { + testCase := testCase + t.Run("", func(t *testing.T) { + gotResult := TagEncode(testCase.inputMap) + if testCase.result != gotResult { + t.Errorf("Expected %s, got %s", testCase.result, gotResult) + } + }) + } +} + // Tests validate the URL path encoder. func TestEncodePath(t *testing.T) { testCases := []struct { @@ -327,6 +403,7 @@ func TestEncodePath(t *testing.T) { {"myurl#link", "myurl%23link"}, {"space in url", "space%20in%20url"}, {"url+path", "url%2Bpath"}, + {"url/path", "url/path"}, } for i, testCase := range testCases { diff --git a/transport.go b/transport.go index 00ee4dad6..fb624854b 100644 --- a/transport.go +++ b/transport.go @@ -38,7 +38,8 @@ var DefaultTransport = func(secure bool) (http.RoundTripper, error) { }).DialContext, MaxIdleConns: 1024, MaxIdleConnsPerHost: 1024, - IdleConnTimeout: 90 * time.Second, + ResponseHeaderTimeout: 60 * time.Second, + IdleConnTimeout: 60 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, // Set this value so that the underlying transport round-tripper diff --git a/utils.go b/utils.go index a6ffedcba..685e98c70 100644 --- a/utils.go +++ b/utils.go @@ -174,6 +174,7 @@ func extractObjMetadata(header http.Header) http.Header { "X-Amz-Object-Lock-Retain-Until-Date", "X-Amz-Object-Lock-Legal-Hold", "X-Amz-Website-Redirect-Location", + "X-Amz-Server-Side-Encryption", "X-Amz-Meta-", // Add new headers to be preserved. // if you add new headers here, please extend @@ -256,6 +257,7 @@ func ToObjectInfo(bucketName string, objectName string, h http.Header) (ObjectIn userMetadata[strings.TrimPrefix(k, "X-Amz-Meta-")] = v[0] } } + userTags := s3utils.TagDecode(h.Get(amzTaggingHeader)) // Save object metadata info. return ObjectInfo{ @@ -270,6 +272,7 @@ func ToObjectInfo(bucketName string, objectName string, h http.Header) (ObjectIn // which are not part of object metadata. Metadata: metadata, UserMetadata: userMetadata, + UserTags: userTags, }, nil } From 44ba45c1aa02cff384a840fe35950b50978bf620 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Tue, 24 Mar 2020 00:04:52 +0530 Subject: [PATCH 124/215] Add Object Retention related flags for CopyObject API (#1254) --- api-compose-object.go | 5 +++++ api-put-object-copy.go | 6 ++++++ api-put-object.go | 2 +- constants.go | 4 ++++ docs/API.md | 4 +++- 5 files changed, 19 insertions(+), 2 deletions(-) diff --git a/api-compose-object.go b/api-compose-object.go index e534b88e4..15e3b6f38 100644 --- a/api-compose-object.go +++ b/api-compose-object.go @@ -63,6 +63,10 @@ type DestInfoOptions struct { // Specifies whether you want to apply a Legal Hold to the copied object. LegalHold LegalHoldStatus + + // Object Retention related fields + Mode RetentionMode + RetainUntilDate time.Time } // Process custom-metadata to remove a `x-amz-meta-` prefix if @@ -111,6 +115,7 @@ func NewDestinationInfo(bucket, object string, sse encrypt.ServerSide, userMeta UserTags: nil, ReplaceTags: false, LegalHold: LegalHoldStatus(""), + Mode: RetentionMode(""), } return DestinationInfo{ bucket: bucket, diff --git a/api-put-object-copy.go b/api-put-object-copy.go index f4db346fb..aa5a36c4d 100644 --- a/api-put-object-copy.go +++ b/api-put-object-copy.go @@ -22,6 +22,7 @@ import ( "io" "io/ioutil" "net/http" + "time" "github.com/minio/minio-go/v6/pkg/encrypt" "github.com/minio/minio-go/v6/pkg/s3utils" @@ -49,6 +50,11 @@ func (c Client) CopyObjectWithProgress(dst DestinationInfo, src SourceInfo, prog header.Set(amzLegalHoldHeader, dst.opts.LegalHold.String()) } + if dst.opts.Mode != RetentionMode("") && !dst.opts.RetainUntilDate.IsZero() { + header.Set(amzLockMode, dst.opts.Mode.String()) + header.Set(amzLockRetainUntil, dst.opts.RetainUntilDate.Format(time.RFC3339)) + } + var err error var size int64 // If progress bar is specified, size should be requested as well initiate a StatObject request. diff --git a/api-put-object.go b/api-put-object.go index 0147a060b..3cad741aa 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -91,7 +91,7 @@ func (opts PutObjectOptions) Header() (header http.Header) { } if opts.Mode != nil { - header.Set("X-Amz-Object-Lock-Mode", opts.Mode.String()) + header.Set(amzLockMode, opts.Mode.String()) } if opts.RetainUntilDate != nil { diff --git a/constants.go b/constants.go index 9d0166250..f7456fa9f 100644 --- a/constants.go +++ b/constants.go @@ -68,4 +68,8 @@ const ( // Object legal hold header amzLegalHoldHeader = "X-Amz-Object-Lock-Legal-Hold" + + // Object retention header + amzLockMode = "X-Amz-Object-Lock-Mode" + amzLockRetainUntil = "X-Amz-Object-Lock-Retain-Until-Date" ) diff --git a/docs/API.md b/docs/API.md index cbc6d53da..63c2c2f53 100644 --- a/docs/API.md +++ b/docs/API.md @@ -950,7 +950,9 @@ __minio.DestInfoOptions__ | `destOpts.UserMetadata` | _map[string]string_ | Map of user meta data to be set on destination object. | | `destOpts.UserTags` | _map[string]string_ | Map of user object tags to be set on destination object. | | `destOpts.ReplaceTags` | _bool_ | Replace object tags of the destination object. | -| `destOpts.LegalHold` | _*minio.LegalHoldStatus_ | LegalHold(En|Dis)abled. | +| `destOpts.LegalHold` | _minio.LegalHoldStatus_ | LegalHold(En|Dis)abled. | +| `destOpts.Mode` | _minio.RetentionMode_ | Retention mode to be set on copied object. | +| `destOpts.RetainUntilDate` | _time.Time_ | Time until object retention should be applied on copied object. | __Example__ From 73284c9cbf05caee67b725c8d0bcb308dec107fc Mon Sep 17 00:00:00 2001 From: "cooper.wu" Date: Wed, 1 Apr 2020 01:49:25 +0800 Subject: [PATCH 125/215] fix: getBucketLocation use virtual hosted style for Aliyun OSS (#1253) --- bucket-cache.go | 16 +++++++++++++--- pkg/s3utils/utils.go | 7 ++++++- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/bucket-cache.go b/bucket-cache.go index 5e46bf9a7..ec5efc314 100644 --- a/bucket-cache.go +++ b/bucket-cache.go @@ -183,11 +183,21 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro } } - targetURL.Path = path.Join(bucketName, "") + "/" - targetURL.RawQuery = urlValues.Encode() + isVirtualHost := s3utils.IsVirtualHostSupported(targetURL, bucketName) + + var urlStr string + + //only suport Aliyun OSS for virtual hosted path, compatible Amazon & Google Endpoint + if isVirtualHost && s3utils.IsAliyunOSSEndpoint(targetURL) { + urlStr = c.endpointURL.Scheme + "://" + bucketName + "." + targetURL.Host + "/?location" + } else { + targetURL.Path = path.Join(bucketName, "") + "/" + targetURL.RawQuery = urlValues.Encode() + urlStr = targetURL.String() + } // Get a new HTTP request for the method. - req, err := http.NewRequest("GET", targetURL.String(), nil) + req, err := http.NewRequest("GET", urlStr, nil) if err != nil { return nil, err } diff --git a/pkg/s3utils/utils.go b/pkg/s3utils/utils.go index d01878dc1..aa96a68fb 100644 --- a/pkg/s3utils/utils.go +++ b/pkg/s3utils/utils.go @@ -78,7 +78,7 @@ func IsVirtualHostSupported(endpointURL url.URL, bucketName string) bool { return false } // Return true for all other cases - return IsAmazonEndpoint(endpointURL) || IsGoogleEndpoint(endpointURL) + return IsAmazonEndpoint(endpointURL) || IsGoogleEndpoint(endpointURL) || IsAliyunOSSEndpoint(endpointURL) } // Refer for region styles - https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region @@ -135,6 +135,11 @@ func GetRegionFromURL(endpointURL url.URL) string { return "" } +// IsAliyunOSSEndpoint - Match if it is exactly Aliyun OSS endpoint. +func IsAliyunOSSEndpoint(endpointURL url.URL) bool { + return strings.HasSuffix(endpointURL.Host, "aliyuncs.com") +} + // IsAmazonEndpoint - Match if it is exactly Amazon S3 endpoint. func IsAmazonEndpoint(endpointURL url.URL) bool { if endpointURL.Host == "s3-external-1.amazonaws.com" || endpointURL.Host == "s3.amazonaws.com" { From eadbcae2a0e66c879f8d1ae54fffbf2ba3c68918 Mon Sep 17 00:00:00 2001 From: Bala FA Date: Wed, 1 Apr 2020 08:37:17 +0000 Subject: [PATCH 126/215] add DisableMultipart flag in PutObjectOptions (#1256) --- api-put-object.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/api-put-object.go b/api-put-object.go index 3cad741aa..eabe788da 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -22,6 +22,7 @@ import ( "context" "crypto/md5" "encoding/base64" + "errors" "fmt" "io" "net/http" @@ -53,6 +54,7 @@ type PutObjectOptions struct { PartSize uint64 LegalHold LegalHoldStatus SendContentMd5 bool + DisableMultipart bool } // getNumThreads - gets the number of threads to be used in the multipart @@ -170,6 +172,10 @@ func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].Part // be uploaded through this operation will be 5TiB. func (c Client) PutObject(bucketName, objectName string, reader io.Reader, objectSize int64, opts PutObjectOptions) (n int64, err error) { + if objectSize < 0 && opts.DisableMultipart { + return 0, errors.New("object size must be provided with disable multipart upload") + } + return c.PutObjectWithContext(context.Background(), bucketName, objectName, reader, objectSize, opts) } @@ -190,7 +196,7 @@ func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName stri } if c.overrideSignerType.IsV2() { - if size >= 0 && size < int64(partSize) { + if size >= 0 && size < int64(partSize) || opts.DisableMultipart { return c.putObject(ctx, bucketName, objectName, reader, size, opts) } return c.putObjectMultipart(ctx, bucketName, objectName, reader, size, opts) @@ -199,7 +205,7 @@ func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName stri return c.putObjectMultipartStreamNoLength(ctx, bucketName, objectName, reader, opts) } - if size < int64(partSize) { + if size < int64(partSize) || opts.DisableMultipart { return c.putObject(ctx, bucketName, objectName, reader, size, opts) } From f170a3b8a9d7cdf7e445478989dbf75f7ff83833 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Thu, 2 Apr 2020 21:57:24 +0000 Subject: [PATCH 127/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 66b06243d..a17b2c1f9 100644 --- a/api.go +++ b/api.go @@ -104,7 +104,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.51" + libraryVersion = "v6.0.52" ) // User Agent should always following the below style. From 94f0286abd36e18113e7a6cb7bbb0dd41c17d067 Mon Sep 17 00:00:00 2001 From: Anis Elleuch Date: Fri, 3 Apr 2020 12:14:30 +0100 Subject: [PATCH 128/215] retry: Quit when the context is cancelled (#1260) The retry goroutine can be stuck when sending a new retry number, this commit will listen for a context cancelltion to properly quit when the retry is canceled from the caller. --- api.go | 9 +++++++++ retry.go | 6 +++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/api.go b/api.go index a17b2c1f9..693de74d5 100644 --- a/api.go +++ b/api.go @@ -630,6 +630,9 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque // Initiate the request. res, err = c.do(req) if err != nil { + if err == context.Canceled || err == context.DeadlineExceeded { + return nil, err + } continue } @@ -709,6 +712,12 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque // For all other cases break out of the retry loop. break } + + // Return an error when retry is canceled or deadlined + if e := retryCtx.Err(); e != nil { + return nil, e + } + return res, err } diff --git a/retry.go b/retry.go index f85a3e76a..6e2826fa6 100644 --- a/retry.go +++ b/retry.go @@ -70,7 +70,11 @@ func (c Client) newRetryTimer(ctx context.Context, maxRetry int, unit time.Durat go func() { defer close(attemptCh) for i := 0; i < maxRetry; i++ { - attemptCh <- i + 1 + select { + case attemptCh <- i + 1: + case <-ctx.Done(): + return + } select { case <-time.After(exponentialBackoffWait(i)): From 965025c971cb02962430f0972b802df1728c6e74 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Fri, 3 Apr 2020 16:44:57 +0530 Subject: [PATCH 129/215] Add support for x-amz-bypass-governance-retention in RemoveObjects (#1258) --- api-object-retention.go | 2 +- api-remove.go | 162 ++++++++++++++++++++++++++-------------- constants.go | 5 +- docs/API.md | 95 +++++++++++++++++++++++ functional_tests.go | 126 ++++++++++++++++++++++++++++++- 5 files changed, 328 insertions(+), 62 deletions(-) diff --git a/api-object-retention.go b/api-object-retention.go index 8aa08f073..822bcc604 100644 --- a/api-object-retention.go +++ b/api-object-retention.go @@ -100,7 +100,7 @@ func (c Client) PutObjectRetention(bucketName, objectName string, opts PutObject if opts.GovernanceBypass { // Set the bypass goverenance retention header - headers.Set("x-amz-bypass-governance-retention", "True") + headers.Set(amzBypassGovernance, "true") } reqMetadata := requestMetadata{ diff --git a/api-remove.go b/api-remove.go index 4c8c335c3..f2879e8b5 100644 --- a/api-remove.go +++ b/api-remove.go @@ -63,7 +63,7 @@ func (c Client) RemoveObject(bucketName, objectName string) error { return c.RemoveObjectWithOptions(bucketName, objectName, RemoveObjectOptions{}) } -// RemoveObjectOptions represents options specified by user for PutObject call +// RemoveObjectOptions represents options specified by user for RemoveObject call type RemoveObjectOptions struct { GovernanceBypass bool VersionID string @@ -92,7 +92,7 @@ func (c Client) RemoveObjectWithOptions(bucketName, objectName string, opts Remo if opts.GovernanceBypass { // Set the bypass goverenance retention header - headers.Set("x-amz-bypass-governance-retention", "True") + headers.Set(amzBypassGovernance, "true") } // Execute DELETE on objectName. resp, err := c.executeMethod(context.Background(), "DELETE", requestMetadata{ @@ -179,71 +179,107 @@ func (c Client) RemoveObjectsWithContext(ctx context.Context, bucketName string, return errorCh } - // Generate and call MultiDelete S3 requests based on entries received from objectsCh - go func(errorCh chan<- RemoveObjectError) { - maxEntries := 1000 - finish := false - urlValues := make(url.Values) - urlValues.Set("delete", "") + go c.removeObjects(ctx, bucketName, objectsCh, errorCh, RemoveObjectsOptions{}) + return errorCh +} + +// RemoveObjectsWithOptionsContext - Identical to RemoveObjects call, but accepts context to +// facilitate request cancellation and options to bypass governance retention +func (c Client) RemoveObjectsWithOptionsContext(ctx context.Context, bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError { + errorCh := make(chan RemoveObjectError, 1) - // Close error channel when Multi delete finishes. + // Validate if bucket name is valid. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + defer close(errorCh) + errorCh <- RemoveObjectError{ + Err: err, + } + return errorCh + } + // Validate objects channel to be properly allocated. + if objectsCh == nil { defer close(errorCh) + errorCh <- RemoveObjectError{ + Err: ErrInvalidArgument("Objects channel cannot be nil"), + } + return errorCh + } - // Loop over entries by 1000 and call MultiDelete requests - for { - if finish { - break - } - count := 0 - var batch []string - - // Try to gather 1000 entries - for object := range objectsCh { - batch = append(batch, object) - if count++; count >= maxEntries { - break - } - } - if count == 0 { - // Multi Objects Delete API doesn't accept empty object list, quit immediately + go c.removeObjects(ctx, bucketName, objectsCh, errorCh, opts) + return errorCh +} + +// Generate and call MultiDelete S3 requests based on entries received from objectsCh +func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh <-chan string, errorCh chan<- RemoveObjectError, opts RemoveObjectsOptions) { + maxEntries := 1000 + finish := false + urlValues := make(url.Values) + urlValues.Set("delete", "") + + // Close error channel when Multi delete finishes. + defer close(errorCh) + + // Loop over entries by 1000 and call MultiDelete requests + for { + if finish { + break + } + count := 0 + var batch []string + + // Try to gather 1000 entries + for object := range objectsCh { + batch = append(batch, object) + if count++; count >= maxEntries { break } - if count < maxEntries { - // We didn't have 1000 entries, so this is the last batch - finish = true - } + } + if count == 0 { + // Multi Objects Delete API doesn't accept empty object list, quit immediately + break + } + if count < maxEntries { + // We didn't have 1000 entries, so this is the last batch + finish = true + } - // Generate remove multi objects XML request - removeBytes := generateRemoveMultiObjectsRequest(batch) - // Execute GET on bucket to list objects. - resp, err := c.executeMethod(ctx, "POST", requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - contentBody: bytes.NewReader(removeBytes), - contentLength: int64(len(removeBytes)), - contentMD5Base64: sumMD5Base64(removeBytes), - contentSHA256Hex: sum256Hex(removeBytes), - }) - if resp != nil { - if resp.StatusCode != http.StatusOK { - e := httpRespToErrorResponse(resp, bucketName, "") - errorCh <- RemoveObjectError{ObjectName: "", Err: e} - } + // Build headers. + headers := make(http.Header) + if opts.GovernanceBypass { + // Set the bypass goverenance retention header + headers.Set(amzBypassGovernance, "true") + } + + // Generate remove multi objects XML request + removeBytes := generateRemoveMultiObjectsRequest(batch) + // Execute GET on bucket to list objects. + resp, err := c.executeMethod(ctx, http.MethodPost, requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: bytes.NewReader(removeBytes), + contentLength: int64(len(removeBytes)), + contentMD5Base64: sumMD5Base64(removeBytes), + contentSHA256Hex: sum256Hex(removeBytes), + customHeader: headers, + }) + if resp != nil { + if resp.StatusCode != http.StatusOK { + e := httpRespToErrorResponse(resp, bucketName, "") + errorCh <- RemoveObjectError{ObjectName: "", Err: e} } - if err != nil { - for _, b := range batch { - errorCh <- RemoveObjectError{ObjectName: b, Err: err} - } - continue + } + if err != nil { + for _, b := range batch { + errorCh <- RemoveObjectError{ObjectName: b, Err: err} } + continue + } - // Process multiobjects remove xml response - processRemoveMultiObjectsResponse(resp.Body, batch, errorCh) + // Process multiobjects remove xml response + processRemoveMultiObjectsResponse(resp.Body, batch, errorCh) - closeResponse(resp) - } - }(errorCh) - return errorCh + closeResponse(resp) + } } // RemoveObjects removes multiple objects from a bucket. @@ -253,6 +289,18 @@ func (c Client) RemoveObjects(bucketName string, objectsCh <-chan string) <-chan return c.RemoveObjectsWithContext(context.Background(), bucketName, objectsCh) } +// RemoveObjectsOptions represents options specified by user for RemoveObjects call +type RemoveObjectsOptions struct { + GovernanceBypass bool +} + +// RemoveObjectsWithOptions removes multiple objects from a bucket. +// The list of objects to remove are received from objectsCh. +// Remove failures are sent back via error channel. +func (c Client) RemoveObjectsWithOptions(bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError { + return c.RemoveObjectsWithOptionsContext(context.Background(), bucketName, objectsCh, opts) +} + // RemoveIncompleteUpload aborts an partially uploaded object. func (c Client) RemoveIncompleteUpload(bucketName, objectName string) error { // Input validation. diff --git a/constants.go b/constants.go index f7456fa9f..78ba5a114 100644 --- a/constants.go +++ b/constants.go @@ -70,6 +70,7 @@ const ( amzLegalHoldHeader = "X-Amz-Object-Lock-Legal-Hold" // Object retention header - amzLockMode = "X-Amz-Object-Lock-Mode" - amzLockRetainUntil = "X-Amz-Object-Lock-Retain-Until-Date" + amzLockMode = "X-Amz-Object-Lock-Mode" + amzLockRetainUntil = "X-Amz-Object-Lock-Retain-Until-Date" + amzBypassGovernance = "X-Amz-Bypass-Governance-Retention" ) diff --git a/docs/API.md b/docs/API.md index 63c2c2f53..9e4403065 100644 --- a/docs/API.md +++ b/docs/API.md @@ -69,6 +69,8 @@ func main() { | | [`FPutObjectWithContext`](#FPutObjectWithContext) | [`FPutObjectWithContext`](#FPutObjectWithContext) | | [`DeleteBucketEncryption`](#DeleteBucketEncryption) | | | [`FGetObjectWithContext`](#FGetObjectWithContext) | [`FGetObjectWithContext`](#FGetObjectWithContext) | | | | | [`RemoveObjectsWithContext`](#RemoveObjectsWithContext) | | | | +| | [`RemoveObjectsWithOptions`](#RemoveObjectsWithOptions) | | | | +| | [`RemoveObjectsWithOptionsContext`](#RemoveObjectsWithOptionsContext) | | | | | | [`RemoveObjectWithOptions`](#RemoveObjectWithOptions) | | | | | | [`PutObjectRetention`](#PutObjectRetention) | | | | | | [`GetObjectRetention`](#GetObjectRetention) | | | | @@ -1253,6 +1255,99 @@ __Parameters__ |`objectName` | _string_ |Name of the object | |`opts` |_minio.PutObjectRetentionOptions_ |Allows user to set options like retention mode, expiry date and version id | + +### RemoveObjectsWithOptions(bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError +*Identical to RemoveObjects operation, but accepts opts for bypassing Governance mode.* + +Parameters + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectsCh` | _chan string_ | Channel of objects to be removed | +|`opts` |_minio.RemoveObjectsOptions_ | Allows user to set options | + +__minio.RemoveObjectsOptions__ + +|Field | Type | Description | +|:--- |:--- | :--- | +| `opts.GovernanceBypass` | _bool_ |Set the bypass governance header to delete an object locked with GOVERNANCE mode| + +__Return Values__ + +|Param |Type |Description | +|:---|:---| :---| +|`errorCh` | _<-chan minio.RemoveObjectError_ | Receive-only channel of errors observed during deletion. | + +```go +objectsCh := make(chan string) + +// Send object names that are needed to be removed to objectsCh +go func() { + defer close(objectsCh) + // List all objects from a bucket-name with a matching prefix. + for object := range minioClient.ListObjects("my-bucketname", "my-prefixname", true, nil) { + if object.Err != nil { + log.Fatalln(object.Err) + } + objectsCh <- object.Key + } +}() + +opts := minio.RemoveObjectsOptions{ + GovernanceBypass: true, +} + +for rErr := range minioClient.RemoveObjectsWithOptions("my-bucketname", objectsCh, opts) { + fmt.Println("Error detected during deletion: ", rErr) +} +``` + + +### RemoveObjectsWithOptionsContext(ctx context.Context, bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError +*Identical to RemoveObjectsWithContext operation, but accepts opts for bypassing Governance mode.* + +Parameters + +|Param |Type |Description | +|:---|:---| :---| +|`ctx` | _context.Context_ |Request context | +|`bucketName` | _string_ |Name of the bucket | +|`objectsCh` | _chan string_ | Channel of objects to be removed | +|`opts` |_minio.RemoveObjectsOptions_ | Allows user to set options | + +__Return Values__ + +|Param |Type |Description | +|:---|:---| :---| +|`errorCh` | _<-chan minio.RemoveObjectError_ | Receive-only channel of errors observed during deletion. | + +```go +objectsCh := make(chan string) +ctx, cancel := context.WithTimeout(context.Background(), 100 * time.Second) +defer cancel() + +// Send object names that are needed to be removed to objectsCh +go func() { + defer close(objectsCh) + // List all objects from a bucket-name with a matching prefix. + for object := range minioClient.ListObjects("my-bucketname", "my-prefixname", true, nil) { + if object.Err != nil { + log.Fatalln(object.Err) + } + objectsCh <- object.Key + } +}() + +opts := minio.RemoveObjectsOptions{ + GovernanceBypass: true, +} + +for rErr := range minioClient.RemoveObjectsWithOptionsContext(ctx, "my-bucketname", objectsCh, opts) { + fmt.Println("Error detected during deletion: ", rErr) +} +``` + __minio.PutObjectRetentionOptions__ |Field | Type | Description | diff --git a/functional_tests.go b/functional_tests.go index d1ec0150c..ca66bc6fd 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -10127,6 +10127,129 @@ func testListObjects() { successLogger(testName, function, args, startTime).Info() } +// Test deleting multiple objects with object retention set in Governance mode +func testRemoveObjectsWithOptions() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "RemoveObjectsWithOptions(bucketName, objectsCh, opts)" + args := map[string]interface{}{ + "bucketName": "", + "objectPrefix": "", + "recursive": "true", + } + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + // Make a new bucket. + err = c.MakeBucketWithObjectLock(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + bufSize := dataFileMap["datafile-129-MB"] + var reader = getDataReader("datafile-129-MB") + defer reader.Close() + + n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) + if err != nil { + log.Fatalln(err) + } + log.Println("Uploaded", objectName, " of size: ", n, "to bucket: ", bucketName, "Successfully.") + + t := time.Date(2030, time.April, 25, 14, 0, 0, 0, time.UTC) + m := minio.RetentionMode(minio.Governance) + opts := minio.PutObjectRetentionOptions{ + GovernanceBypass: false, + RetainUntilDate: &t, + Mode: &m, + } + err = c.PutObjectRetention(bucketName, objectName, opts) + if err != nil { + log.Fatalln(err) + } + + objectsCh := make(chan string) + // Send object names that are needed to be removed to objectsCh + go func() { + defer close(objectsCh) + // List all objects from a bucket-name with a matching prefix. + for object := range c.ListObjects(bucketName, "", true, nil) { + if object.Err != nil { + log.Fatalln(object.Err) + } + objectsCh <- object.Key + } + }() + + for rErr := range c.RemoveObjects(bucketName, objectsCh) { + // Error is expected here because Retention is set on the object + // and RemoveObjects is called without Bypass Governance + if rErr.Err == nil { + logError(testName, function, args, startTime, "", "Expected error during deletion", nil) + return + } + } + + objectsCh1 := make(chan string) + + // Send object names that are needed to be removed to objectsCh + go func() { + defer close(objectsCh1) + // List all objects from a bucket-name with a matching prefix. + for object := range c.ListObjects(bucketName, "", true, nil) { + if object.Err != nil { + log.Fatalln(object.Err) + } + objectsCh1 <- object.Key + } + }() + + opts1 := minio.RemoveObjectsOptions{ + GovernanceBypass: true, + } + + for rErr := range c.RemoveObjectsWithOptions(bucketName, objectsCh1, opts1) { + // Error is not expected here because Retention is set on the object + // and RemoveObjects is called with Bypass Governance + logError(testName, function, args, startTime, "", "Error detected during deletion", rErr.Err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + // Convert string to bool and always return false if any error func mustParseBool(str string) bool { b, err := strconv.ParseBool(str) @@ -10193,15 +10316,14 @@ func main() { testGetObjectWithContext() testFPutObjectWithContext() testFGetObjectWithContext() - testGetObjectACLWithContext() - testPutObjectWithContext() testStorageClassMetadataPutObject() testStorageClassInvalidMetadataPutObject() testStorageClassMetadataCopyObject() testPutObjectWithContentLanguage() testListObjects() + testRemoveObjectsWithOptions() // SSE-C tests will only work over TLS connection. if tls { From 74bf9a77866de9564af5528433bb627361c05ff8 Mon Sep 17 00:00:00 2001 From: Vincent Besanceney Date: Fri, 3 Apr 2020 13:15:37 +0200 Subject: [PATCH 130/215] Export identifiers for shared credentials providers (#1255) This makes it possible to build a chain provider with this kind of providers, e.g.: ``` creds := credentials.NewChainCredentials( []credentials.Provider{ &credentials.FileAWSCredentials{ Filename: filename, Profile: profile }, }) ``` --- pkg/credentials/file_aws_credentials.go | 26 ++++++++++++------------- pkg/credentials/file_minio_client.go | 26 ++++++++++++------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/pkg/credentials/file_aws_credentials.go b/pkg/credentials/file_aws_credentials.go index ff07bc55b..ccc8251f4 100644 --- a/pkg/credentials/file_aws_credentials.go +++ b/pkg/credentials/file_aws_credentials.go @@ -36,12 +36,12 @@ type FileAWSCredentials struct { // env value is empty will default to current user's home directory. // Linux/OSX: "$HOME/.aws/credentials" // Windows: "%USERPROFILE%\.aws\credentials" - filename string + Filename string // AWS Profile to extract credentials from the shared credentials file. If empty // will default to environment variable "AWS_PROFILE" or "default" if // environment variable is also not set. - profile string + Profile string // retrieved states if the credentials have been successfully retrieved. retrieved bool @@ -51,34 +51,34 @@ type FileAWSCredentials struct { // wrapping the Profile file provider. func NewFileAWSCredentials(filename string, profile string) *Credentials { return New(&FileAWSCredentials{ - filename: filename, - profile: profile, + Filename: filename, + Profile: profile, }) } // Retrieve reads and extracts the shared credentials from the current // users home directory. func (p *FileAWSCredentials) Retrieve() (Value, error) { - if p.filename == "" { - p.filename = os.Getenv("AWS_SHARED_CREDENTIALS_FILE") - if p.filename == "" { + if p.Filename == "" { + p.Filename = os.Getenv("AWS_SHARED_CREDENTIALS_FILE") + if p.Filename == "" { homeDir, err := homedir.Dir() if err != nil { return Value{}, err } - p.filename = filepath.Join(homeDir, ".aws", "credentials") + p.Filename = filepath.Join(homeDir, ".aws", "credentials") } } - if p.profile == "" { - p.profile = os.Getenv("AWS_PROFILE") - if p.profile == "" { - p.profile = "default" + if p.Profile == "" { + p.Profile = os.Getenv("AWS_PROFILE") + if p.Profile == "" { + p.Profile = "default" } } p.retrieved = false - iniProfile, err := loadProfile(p.filename, p.profile) + iniProfile, err := loadProfile(p.Filename, p.Profile) if err != nil { return Value{}, err } diff --git a/pkg/credentials/file_minio_client.go b/pkg/credentials/file_minio_client.go index 117ceb6ef..8adb5dcb1 100644 --- a/pkg/credentials/file_minio_client.go +++ b/pkg/credentials/file_minio_client.go @@ -38,12 +38,12 @@ type FileMinioClient struct { // env value is empty will default to current user's home directory. // Linux/OSX: "$HOME/.mc/config.json" // Windows: "%USERALIAS%\mc\config.json" - filename string + Filename string // MinIO Alias to extract credentials from the shared credentials file. If empty // will default to environment variable "MINIO_ALIAS" or "default" if // environment variable is also not set. - alias string + Alias string // retrieved states if the credentials have been successfully retrieved. retrieved bool @@ -53,39 +53,39 @@ type FileMinioClient struct { // wrapping the Alias file provider. func NewFileMinioClient(filename string, alias string) *Credentials { return New(&FileMinioClient{ - filename: filename, - alias: alias, + Filename: filename, + Alias: alias, }) } // Retrieve reads and extracts the shared credentials from the current // users home directory. func (p *FileMinioClient) Retrieve() (Value, error) { - if p.filename == "" { + if p.Filename == "" { if value, ok := os.LookupEnv("MINIO_SHARED_CREDENTIALS_FILE"); ok { - p.filename = value + p.Filename = value } else { homeDir, err := homedir.Dir() if err != nil { return Value{}, err } - p.filename = filepath.Join(homeDir, ".mc", "config.json") + p.Filename = filepath.Join(homeDir, ".mc", "config.json") if runtime.GOOS == "windows" { - p.filename = filepath.Join(homeDir, "mc", "config.json") + p.Filename = filepath.Join(homeDir, "mc", "config.json") } } } - if p.alias == "" { - p.alias = os.Getenv("MINIO_ALIAS") - if p.alias == "" { - p.alias = "s3" + if p.Alias == "" { + p.Alias = os.Getenv("MINIO_ALIAS") + if p.Alias == "" { + p.Alias = "s3" } } p.retrieved = false - hostCfg, err := loadAlias(p.filename, p.alias) + hostCfg, err := loadAlias(p.Filename, p.Alias) if err != nil { return Value{}, err } From 73469ba42c49c64ce966cafd90acf3934d3c908f Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 3 Apr 2020 04:20:26 -0700 Subject: [PATCH 131/215] fix: go mod tidy --- go.mod | 2 +- go.sum | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 2dc189d1a..fe6d38bc6 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/dustin/go-humanize v1.0.0 // indirect github.com/minio/sha256-simd v0.1.1 github.com/mitchellh/go-homedir v1.1.0 - github.com/sirupsen/logrus v1.4.2 // indirect + github.com/sirupsen/logrus v1.5.0 // indirect github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f golang.org/x/net v0.0.0-20190522155817-f3200d17e092 diff --git a/go.sum b/go.sum index cd02277ed..57de3a2b9 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,3 @@ -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -11,16 +10,13 @@ github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKU github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q= +github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo= From edb71e2e1f0cf8733c906dba3a5806dffd081eed Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Fri, 3 Apr 2020 23:59:05 +0000 Subject: [PATCH 132/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 693de74d5..dce2ae1e2 100644 --- a/api.go +++ b/api.go @@ -104,7 +104,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.52" + libraryVersion = "v6.0.53" ) // User Agent should always following the below style. From 4c090d425872cd76e9413b0afdefb8a769c1cd9c Mon Sep 17 00:00:00 2001 From: Daniel Valdivia Date: Tue, 7 Apr 2020 16:03:53 -0700 Subject: [PATCH 133/215] add functions to remove notifications by prefix/suffix (#1262) --- bucket-notification.go | 109 +++++++++++++++++- bucket-notification_test.go | 217 ++++++++++++++++++++++++++++++++++++ 2 files changed, 324 insertions(+), 2 deletions(-) create mode 100644 bucket-notification_test.go diff --git a/bucket-notification.go b/bucket-notification.go index 557e93c1c..277c2e11d 100644 --- a/bucket-notification.go +++ b/bucket-notification.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,7 +19,7 @@ package minio import ( "encoding/xml" - + "fmt" "github.com/minio/minio-go/v6/pkg/set" ) @@ -137,6 +137,66 @@ func (t *NotificationConfig) AddFilterPrefix(prefix string) { t.Filter.S3Key.FilterRules = append(t.Filter.S3Key.FilterRules, newFilterRule) } +// EqualNotificationEventTypeList tells whether a and b contain the same events +func EqualNotificationEventTypeList(a, b []NotificationEventType) bool { + if len(a) != len(b) { + return false + } + setA := set.NewStringSet() + for _, i := range a { + setA.Add(string(i)) + } + + setB := set.NewStringSet() + for _, i := range b { + setB.Add(string(i)) + } + + return setA.Difference(setB).IsEmpty() +} + +// EqualFilterRuleList tells whether a and b contain the same filters +func EqualFilterRuleList(a, b []FilterRule) bool { + if len(a) != len(b) { + return false + } + + setA := set.NewStringSet() + for _, i := range a { + setA.Add(fmt.Sprintf("%s-%s", i.Name, i.Value)) + } + + setB := set.NewStringSet() + for _, i := range b { + setB.Add(fmt.Sprintf("%s-%s", i.Name, i.Value)) + } + + return setA.Difference(setB).IsEmpty() +} + +// Equal returns whether this `NotificationConfig` is equal to another defined by the passed parameters +func (t *NotificationConfig) Equal(arn string, events []NotificationEventType, prefix, suffix string) bool { + // if it's not the same ARN, return immediately. + if t.Arn.String() != arn { + return false + } + //Compare events + passEvents := EqualNotificationEventTypeList(t.Events, events) + + //Compare filters + var newFilter []FilterRule + if prefix != "" { + newFilter = append(newFilter, FilterRule{Name: "prefix", Value: prefix}) + } + if suffix != "" { + newFilter = append(newFilter, FilterRule{Name: "suffix", Value: suffix}) + } + + passFilters := EqualFilterRuleList(t.Filter.S3Key.FilterRules, newFilter) + // if it matches events and filters, mark the index for deletion + return passEvents && passFilters +} + // TopicConfig carries one single topic notification configuration type TopicConfig struct { NotificationConfig @@ -250,6 +310,21 @@ func (b *BucketNotification) RemoveTopicByArn(arn Arn) { b.TopicConfigs = topics } +// RemoveTopicByArnEventsPrefixSuffix removes a topic configuration that match the exact specified ARN, events, prefix and suffix +func (b *BucketNotification) RemoveTopicByArnEventsPrefixSuffix(arn Arn, events []NotificationEventType, prefix, suffix string) { + removeIndex := -1 + for i, v := range b.TopicConfigs { + // if it matches events and filters, mark the index for deletion + if v.NotificationConfig.Equal(arn.String(), events, prefix, suffix) { + removeIndex = i + break // since we have at most one matching config + } + } + if removeIndex >= 0 { + b.TopicConfigs = append(b.TopicConfigs[:removeIndex], b.TopicConfigs[removeIndex+1:]...) + } +} + // RemoveQueueByArn removes all queue configurations that match the exact specified ARN func (b *BucketNotification) RemoveQueueByArn(arn Arn) { var queues []QueueConfig @@ -261,6 +336,21 @@ func (b *BucketNotification) RemoveQueueByArn(arn Arn) { b.QueueConfigs = queues } +// RemoveQueueByArnEventsPrefixSuffix removes a queue configuration that match the exact specified ARN, events, prefix and suffix +func (b *BucketNotification) RemoveQueueByArnEventsPrefixSuffix(arn Arn, events []NotificationEventType, prefix, suffix string) { + removeIndex := -1 + for i, v := range b.QueueConfigs { + // if it matches events and filters, mark the index for deletion + if v.NotificationConfig.Equal(arn.String(), events, prefix, suffix) { + removeIndex = i + break // since we have at most one matching config + } + } + if removeIndex >= 0 { + b.QueueConfigs = append(b.QueueConfigs[:removeIndex], b.QueueConfigs[removeIndex+1:]...) + } +} + // RemoveLambdaByArn removes all lambda configurations that match the exact specified ARN func (b *BucketNotification) RemoveLambdaByArn(arn Arn) { var lambdas []LambdaConfig @@ -271,3 +361,18 @@ func (b *BucketNotification) RemoveLambdaByArn(arn Arn) { } b.LambdaConfigs = lambdas } + +// RemoveLambdaByArnEventsPrefixSuffix removes a topic configuration that match the exact specified ARN, events, prefix and suffix +func (b *BucketNotification) RemoveLambdaByArnEventsPrefixSuffix(arn Arn, events []NotificationEventType, prefix, suffix string) { + removeIndex := -1 + for i, v := range b.LambdaConfigs { + // if it matches events and filters, mark the index for deletion + if v.NotificationConfig.Equal(arn.String(), events, prefix, suffix) { + removeIndex = i + break // since we have at most one matching config + } + } + if removeIndex >= 0 { + b.LambdaConfigs = append(b.LambdaConfigs[:removeIndex], b.LambdaConfigs[removeIndex+1:]...) + } +} diff --git a/bucket-notification_test.go b/bucket-notification_test.go new file mode 100644 index 000000000..a0c5ef30f --- /dev/null +++ b/bucket-notification_test.go @@ -0,0 +1,217 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import "testing" + +func TestEqualNotificationEventTypeList(t *testing.T) { + type args struct { + a []NotificationEventType + b []NotificationEventType + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "same order", + args: args{ + a: []NotificationEventType{ObjectCreatedAll, ObjectAccessedAll}, + b: []NotificationEventType{ObjectCreatedAll, ObjectAccessedAll}, + }, + want: true, + }, + { + name: "different order", + args: args{ + a: []NotificationEventType{ObjectCreatedAll, ObjectAccessedAll}, + b: []NotificationEventType{ObjectAccessedAll, ObjectCreatedAll}, + }, + want: true, + }, + { + name: "not equal", + args: args{ + a: []NotificationEventType{ObjectCreatedAll, ObjectAccessedAll}, + b: []NotificationEventType{ObjectRemovedAll, ObjectAccessedAll}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := EqualNotificationEventTypeList(tt.args.a, tt.args.b); got != tt.want { + t.Errorf("EqualNotificationEventTypeList() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestEqualFilterRuleList(t *testing.T) { + type args struct { + a []FilterRule + b []FilterRule + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "same order", + args: args{ + a: []FilterRule{{Name: "prefix", Value: "prefix1"}, {Name: "suffix", Value: "suffix1"}}, + b: []FilterRule{{Name: "prefix", Value: "prefix1"}, {Name: "suffix", Value: "suffix1"}}, + }, + want: true, + }, + { + name: "different order", + args: args{ + a: []FilterRule{{Name: "prefix", Value: "prefix1"}, {Name: "suffix", Value: "suffix1"}}, + b: []FilterRule{{Name: "suffix", Value: "suffix1"}, {Name: "prefix", Value: "prefix1"}}, + }, + want: true, + }, + { + name: "not equal", + args: args{ + a: []FilterRule{{Name: "prefix", Value: "prefix1"}, {Name: "suffix", Value: "suffix1"}}, + b: []FilterRule{{Name: "prefix", Value: "prefix2"}, {Name: "suffix", Value: "suffix1"}}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := EqualFilterRuleList(tt.args.a, tt.args.b); got != tt.want { + t.Errorf("EqualFilterRuleList() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNotificationConfig_Equal(t *testing.T) { + type fields struct { + ID string + Arn Arn + Events []NotificationEventType + Filter *Filter + } + type args struct { + arn string + events []NotificationEventType + prefix string + suffix string + } + tests := []struct { + name string + fields fields + args args + want bool + }{ + { + name: "same order", + fields: fields{ + Arn: NewArn("minio", "sqs", "", "1", "postgresql"), + Events: []NotificationEventType{ObjectCreatedAll, ObjectAccessedAll}, + Filter: &Filter{ + S3Key: S3Key{ + FilterRules: []FilterRule{{Name: "prefix", Value: "prefix1"}, {Name: "suffix", Value: "suffix1"}}, + }, + }, + }, + args: args{ + arn: "arn:minio:sqs::1:postgresql", + events: []NotificationEventType{ObjectCreatedAll, ObjectAccessedAll}, + prefix: "prefix1", + suffix: "suffix1", + }, + want: true, + }, + { + name: "different order", + fields: fields{ + Arn: NewArn("minio", "sqs", "", "1", "postgresql"), + Events: []NotificationEventType{ObjectAccessedAll, ObjectCreatedAll}, + Filter: &Filter{ + S3Key: S3Key{ + FilterRules: []FilterRule{{Name: "suffix", Value: "suffix1"}, {Name: "prefix", Value: "prefix1"}}, + }, + }, + }, + args: args{ + arn: "arn:minio:sqs::1:postgresql", + events: []NotificationEventType{ObjectCreatedAll, ObjectAccessedAll}, + prefix: "prefix1", + suffix: "suffix1", + }, + want: true, + }, + { + name: "not equal", + fields: fields{ + Arn: NewArn("minio", "sqs", "", "1", "postgresql"), + Events: []NotificationEventType{ObjectAccessedAll}, + Filter: &Filter{ + S3Key: S3Key{ + FilterRules: []FilterRule{{Name: "suffix", Value: "suffix1"}, {Name: "prefix", Value: "prefix1"}}, + }, + }, + }, + args: args{ + arn: "arn:minio:sqs::1:postgresql", + events: []NotificationEventType{ObjectCreatedAll, ObjectAccessedAll}, + prefix: "prefix1", + suffix: "suffix1", + }, + want: false, + }, + { + name: "different arn", + fields: fields{ + Arn: NewArn("minio", "sqs", "", "2", "postgresql"), + Events: []NotificationEventType{ObjectAccessedAll}, + Filter: &Filter{ + S3Key: S3Key{ + FilterRules: []FilterRule{{Name: "suffix", Value: "suffix1"}, {Name: "prefix", Value: "prefix1"}}, + }, + }, + }, + args: args{ + arn: "arn:minio:sqs::1:postgresql", + events: []NotificationEventType{ObjectCreatedAll, ObjectAccessedAll}, + prefix: "prefix1", + suffix: "suffix1", + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + nc := &NotificationConfig{ + ID: tt.fields.ID, + Arn: tt.fields.Arn, + Events: tt.fields.Events, + Filter: tt.fields.Filter, + } + if got := nc.Equal(tt.args.arn, tt.args.events, tt.args.prefix, tt.args.suffix); got != tt.want { + t.Errorf("Equal() = %v, want %v", got, tt.want) + } + }) + } +} From 8bb083d4abf4d0dbdcc409b33661f65b7e1fde51 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 13 Apr 2020 12:43:08 -0700 Subject: [PATCH 134/215] Add AssumeRole support for rotating credentials (#1265) --- api-presigned.go | 8 +- api.go | 14 +- bucket-cache.go | 6 +- bucket-cache_test.go | 6 +- core_test.go | 2 +- pkg/credentials/assume_role.go | 214 ++++++++++++++++++ .../request-signature-streaming.go | 14 +- .../request-signature-streaming_test.go | 2 +- .../request-signature-v2.go | 2 +- .../request-signature-v2_test.go | 2 +- .../request-signature-v4.go | 95 ++++---- .../request-signature-v4_test.go | 2 +- .../request-signature_test.go | 2 +- pkg/{s3signer => signer}/test-utils_test.go | 2 +- pkg/{s3signer => signer}/utils.go | 2 +- pkg/{s3signer => signer}/utils_test.go | 2 +- 16 files changed, 296 insertions(+), 79 deletions(-) create mode 100644 pkg/credentials/assume_role.go rename pkg/{s3signer => signer}/request-signature-streaming.go (97%) rename pkg/{s3signer => signer}/request-signature-streaming_test.go (99%) rename pkg/{s3signer => signer}/request-signature-v2.go (99%) rename pkg/{s3signer => signer}/request-signature-v2_test.go (98%) rename pkg/{s3signer => signer}/request-signature-v4.go (79%) rename pkg/{s3signer => signer}/request-signature-v4_test.go (99%) rename pkg/{s3signer => signer}/request-signature_test.go (99%) rename pkg/{s3signer => signer}/test-utils_test.go (99%) rename pkg/{s3signer => signer}/utils.go (99%) rename pkg/{s3signer => signer}/utils_test.go (99%) diff --git a/api-presigned.go b/api-presigned.go index e2d68b0ec..2e27c3f97 100644 --- a/api-presigned.go +++ b/api-presigned.go @@ -23,8 +23,8 @@ import ( "net/url" "time" - "github.com/minio/minio-go/v6/pkg/s3signer" "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/signer" ) // presignURL - Returns a presigned URL for an input 'method'. @@ -157,7 +157,7 @@ func (c Client) PresignedPostPolicy(p *PostPolicy) (u *url.URL, formData map[str p.formData["AWSAccessKeyId"] = accessKeyID } // Sign the policy. - p.formData["signature"] = s3signer.PostPresignSignatureV2(policyBase64, secretAccessKey) + p.formData["signature"] = signer.PostPresignSignatureV2(policyBase64, secretAccessKey) return u, p.formData, nil } @@ -180,7 +180,7 @@ func (c Client) PresignedPostPolicy(p *PostPolicy) (u *url.URL, formData map[str } // Add a credential policy. - credential := s3signer.GetCredential(accessKeyID, location, t) + credential := signer.GetCredential(accessKeyID, location, t, signer.ServiceTypeS3) if err = p.addNewPolicy(policyCondition{ matchType: "eq", condition: "$x-amz-credential", @@ -210,6 +210,6 @@ func (c Client) PresignedPostPolicy(p *PostPolicy) (u *url.URL, formData map[str if sessionToken != "" { p.formData["x-amz-security-token"] = sessionToken } - p.formData["x-amz-signature"] = s3signer.PostPresignSignatureV4(policyBase64, t, secretAccessKey, location) + p.formData["x-amz-signature"] = signer.PostPresignSignatureV4(policyBase64, t, secretAccessKey, location) return u, p.formData, nil } diff --git a/api.go b/api.go index dce2ae1e2..cb094ea32 100644 --- a/api.go +++ b/api.go @@ -43,8 +43,8 @@ import ( "golang.org/x/net/publicsuffix" "github.com/minio/minio-go/v6/pkg/credentials" - "github.com/minio/minio-go/v6/pkg/s3signer" "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/signer" ) // Client implements Amazon S3 compatible methods. @@ -271,7 +271,7 @@ func (c *Client) redirectHeaders(req *http.Request, via []*http.Request) error { case signerType.IsV2(): return errors.New("signature V2 cannot support redirection") case signerType.IsV4(): - s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, getDefaultLocation(*c.endpointURL, region)) + signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, getDefaultLocation(*c.endpointURL, region)) } } return nil @@ -792,10 +792,10 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R } if signerType.IsV2() { // Presign URL with signature v2. - req = s3signer.PreSignV2(*req, accessKeyID, secretAccessKey, metadata.expires, isVirtualHost) + req = signer.PreSignV2(*req, accessKeyID, secretAccessKey, metadata.expires, isVirtualHost) } else if signerType.IsV4() { // Presign URL with signature v4. - req = s3signer.PreSignV4(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires) + req = signer.PreSignV4(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires) } return req, nil } @@ -838,12 +838,12 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R switch { case signerType.IsV2(): // Add signature version '2' authorization header. - req = s3signer.SignV2(*req, accessKeyID, secretAccessKey, isVirtualHost) + req = signer.SignV2(*req, accessKeyID, secretAccessKey, isVirtualHost) case metadata.objectName != "" && metadata.queryValues == nil && method == "PUT" && metadata.customHeader.Get("X-Amz-Copy-Source") == "" && !c.secure: // Streaming signature is used by default for a PUT object request. Additionally we also // look if the initialized client is secure, if yes then we don't need to perform // streaming signature. - req = s3signer.StreamingSignV4(req, accessKeyID, + req = signer.StreamingSignV4(req, accessKeyID, secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC()) default: // Set sha256 sum for signature calculation only with signature version '4'. @@ -854,7 +854,7 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R req.Header.Set("X-Amz-Content-Sha256", shaHeader) // Add signature version '4' authorization header. - req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, location) + req = signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, location) } // Return request. diff --git a/bucket-cache.go b/bucket-cache.go index ec5efc314..6727840ae 100644 --- a/bucket-cache.go +++ b/bucket-cache.go @@ -25,8 +25,8 @@ import ( "sync" "github.com/minio/minio-go/v6/pkg/credentials" - "github.com/minio/minio-go/v6/pkg/s3signer" "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/signer" ) // bucketLocationCache - Provides simple mechanism to hold bucket @@ -236,7 +236,7 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro if signerType.IsV2() { // Get Bucket Location calls should be always path style isVirtualHost := false - req = s3signer.SignV2(*req, accessKeyID, secretAccessKey, isVirtualHost) + req = signer.SignV2(*req, accessKeyID, secretAccessKey, isVirtualHost) return req, nil } @@ -247,6 +247,6 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro } req.Header.Set("X-Amz-Content-Sha256", contentSha256) - req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1") + req = signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1") return req, nil } diff --git a/bucket-cache_test.go b/bucket-cache_test.go index e095a5c0e..3c14a8b99 100644 --- a/bucket-cache_test.go +++ b/bucket-cache_test.go @@ -28,7 +28,7 @@ import ( "testing" "github.com/minio/minio-go/v6/pkg/credentials" - "github.com/minio/minio-go/v6/pkg/s3signer" + "github.com/minio/minio-go/v6/pkg/signer" ) // Test validates `newBucketLocationCache`. @@ -119,9 +119,9 @@ func TestGetBucketLocationRequest(t *testing.T) { contentSha256 = unsignedPayload } req.Header.Set("X-Amz-Content-Sha256", contentSha256) - req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1") + req = signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1") case signerType.IsV2(): - req = s3signer.SignV2(*req, accessKeyID, secretAccessKey, false) + req = signer.SignV2(*req, accessKeyID, secretAccessKey, false) } return req, nil diff --git a/core_test.go b/core_test.go index e22d5ae34..467c949f3 100644 --- a/core_test.go +++ b/core_test.go @@ -456,7 +456,7 @@ func TestCoreCopyObject(t *testing.T) { t.Fatal("Error:", err, bucketName, objectName, destBucketName, destObjectName) } if cobjInfo.ETag != objInfo.ETag { - t.Fatalf("Error: expected etag to be same as source object %s, but found different etag :%s", objInfo.ETag, cobjInfo.ETag) + t.Fatalf("Error: expected etag to be same as source object %s, but found different etag %s", objInfo.ETag, cobjInfo.ETag) } // Attempt to read from destBucketName and object name. diff --git a/pkg/credentials/assume_role.go b/pkg/credentials/assume_role.go new file mode 100644 index 000000000..f3702d8d6 --- /dev/null +++ b/pkg/credentials/assume_role.go @@ -0,0 +1,214 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "encoding/hex" + "encoding/xml" + "errors" + "io" + "io/ioutil" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/minio/minio-go/v6/pkg/signer" + sha256 "github.com/minio/sha256-simd" +) + +// AssumeRoleResponse contains the result of successful AssumeRole request. +type AssumeRoleResponse struct { + XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleResponse" json:"-"` + + Result AssumeRoleResult `xml:"AssumeRoleResult"` + ResponseMetadata struct { + RequestID string `xml:"RequestId,omitempty"` + } `xml:"ResponseMetadata,omitempty"` +} + +// AssumeRoleResult - Contains the response to a successful AssumeRole +// request, including temporary credentials that can be used to make +// MinIO API requests. +type AssumeRoleResult struct { + // The identifiers for the temporary security credentials that the operation + // returns. + AssumedRoleUser AssumedRoleUser `xml:",omitempty"` + + // The temporary security credentials, which include an access key ID, a secret + // access key, and a security (or session) token. + // + // Note: The size of the security token that STS APIs return is not fixed. We + // strongly recommend that you make no assumptions about the maximum size. As + // of this writing, the typical size is less than 4096 bytes, but that can vary. + // Also, future updates to AWS might require larger sizes. + Credentials struct { + AccessKey string `xml:"AccessKeyId" json:"accessKey,omitempty"` + SecretKey string `xml:"SecretAccessKey" json:"secretKey,omitempty"` + Expiration time.Time `xml:"Expiration" json:"expiration,omitempty"` + SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty"` + } `xml:",omitempty"` + + // A percentage value that indicates the size of the policy in packed form. + // The service rejects any policy with a packed size greater than 100 percent, + // which means the policy exceeded the allowed space. + PackedPolicySize int `xml:",omitempty"` +} + +// A STSAssumeRole retrieves credentials from MinIO service, and keeps track if +// those credentials are expired. +type STSAssumeRole struct { + Expiry + + // Required http Client to use when connecting to MinIO STS service. + Client *http.Client + + // STS endpoint to fetch STS credentials. + STSEndpoint string + + // various options for this request. + Options STSAssumeRoleOptions +} + +// STSAssumeRoleOptions collection of various input options +// to obtain AssumeRole credentials. +type STSAssumeRoleOptions struct { + // Mandatory inputs. + AccessKey string + SecretKey string + + Location string // Optional commonly needed with AWS STS. + DurationSeconds int // Optional defaults to 1 hour. + + // Optional only valid if using with AWS STS + RoleARN string + RoleSessionName string +} + +// NewSTSAssumeRole returns a pointer to a new +// Credentials object wrapping the STSAssumeRole. +func NewSTSAssumeRole(stsEndpoint string, opts STSAssumeRoleOptions) (*Credentials, error) { + if stsEndpoint == "" { + return nil, errors.New("STS endpoint cannot be empty") + } + if opts.AccessKey == "" || opts.SecretKey == "" { + return nil, errors.New("AssumeRole credentials access/secretkey is mandatory") + } + return New(&STSAssumeRole{ + Client: &http.Client{ + Transport: http.DefaultTransport, + }, + STSEndpoint: stsEndpoint, + Options: opts, + }), nil +} + +const defaultDurationSeconds = 3600 + +// closeResponse close non nil response with any response Body. +// convenient wrapper to drain any remaining data on response body. +// +// Subsequently this allows golang http RoundTripper +// to re-use the same connection for future requests. +func closeResponse(resp *http.Response) { + // Callers should close resp.Body when done reading from it. + // If resp.Body is not closed, the Client's underlying RoundTripper + // (typically Transport) may not be able to re-use a persistent TCP + // connection to the server for a subsequent "keep-alive" request. + if resp != nil && resp.Body != nil { + // Drain any remaining Body and then close the connection. + // Without this closing connection would disallow re-using + // the same connection for future uses. + // - http://stackoverflow.com/a/17961593/4465767 + io.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + } +} + +func getAssumeRoleCredentials(clnt *http.Client, endpoint string, opts STSAssumeRoleOptions) (AssumeRoleResponse, error) { + v := url.Values{} + v.Set("Action", "AssumeRole") + v.Set("Version", "2011-06-15") + if opts.RoleARN != "" { + v.Set("RoleArn", opts.RoleARN) + } + if opts.RoleSessionName != "" { + v.Set("RoleSessionName", opts.RoleSessionName) + } + if opts.DurationSeconds > defaultDurationSeconds { + v.Set("DurationSeconds", strconv.Itoa(opts.DurationSeconds)) + } else { + v.Set("DurationSeconds", strconv.Itoa(defaultDurationSeconds)) + } + + u, err := url.Parse(endpoint) + if err != nil { + return AssumeRoleResponse{}, err + } + u.Path = "/" + + postBody := strings.NewReader(v.Encode()) + hash := sha256.New() + if _, err = io.Copy(hash, postBody); err != nil { + return AssumeRoleResponse{}, err + } + postBody.Seek(0, 0) + + req, err := http.NewRequest("POST", u.String(), postBody) + if err != nil { + return AssumeRoleResponse{}, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(hash.Sum(nil))) + req = signer.SignV4STS(*req, opts.AccessKey, opts.SecretKey, opts.Location) + + resp, err := clnt.Do(req) + if err != nil { + return AssumeRoleResponse{}, err + } + defer closeResponse(resp) + if resp.StatusCode != http.StatusOK { + return AssumeRoleResponse{}, errors.New(resp.Status) + } + + a := AssumeRoleResponse{} + if err = xml.NewDecoder(resp.Body).Decode(&a); err != nil { + return AssumeRoleResponse{}, err + } + return a, nil +} + +// Retrieve retrieves credentials from the MinIO service. +// Error will be returned if the request fails. +func (m *STSAssumeRole) Retrieve() (Value, error) { + a, err := getAssumeRoleCredentials(m.Client, m.STSEndpoint, m.Options) + if err != nil { + return Value{}, err + } + + // Expiry window is set to 10secs. + m.SetExpiration(a.Result.Credentials.Expiration, DefaultExpiryWindow) + + return Value{ + AccessKeyID: a.Result.Credentials.AccessKey, + SecretAccessKey: a.Result.Credentials.SecretKey, + SessionToken: a.Result.Credentials.SessionToken, + SignerType: SignatureV4, + }, nil +} diff --git a/pkg/s3signer/request-signature-streaming.go b/pkg/signer/request-signature-streaming.go similarity index 97% rename from pkg/s3signer/request-signature-streaming.go rename to pkg/signer/request-signature-streaming.go index 810b47c4b..7b2ca91d1 100644 --- a/pkg/s3signer/request-signature-streaming.go +++ b/pkg/signer/request-signature-streaming.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package s3signer +package signer import ( "bytes" @@ -82,7 +82,7 @@ func buildChunkStringToSign(t time.Time, region, previousSig string, chunkData [ stringToSignParts := []string{ streamingPayloadHdr, t.Format(iso8601DateFormat), - getScope(region, t), + getScope(region, t, ServiceTypeS3), previousSig, emptySHA256, hex.EncodeToString(sum256(chunkData)), @@ -118,19 +118,19 @@ func buildChunkSignature(chunkData []byte, reqTime time.Time, region, chunkStringToSign := buildChunkStringToSign(reqTime, region, previousSignature, chunkData) - signingKey := getSigningKey(secretAccessKey, region, reqTime) + signingKey := getSigningKey(secretAccessKey, region, reqTime, ServiceTypeS3) return getSignature(signingKey, chunkStringToSign) } // getSeedSignature - returns the seed signature for a given request. func (s *StreamingReader) setSeedSignature(req *http.Request) { // Get canonical request - canonicalRequest := getCanonicalRequest(*req, ignoredStreamingHeaders) + canonicalRequest := getCanonicalRequest(*req, ignoredStreamingHeaders, getHashedPayload(*req)) // Get string to sign from canonical request. - stringToSign := getStringToSignV4(s.reqTime, s.region, canonicalRequest) + stringToSign := getStringToSignV4(s.reqTime, s.region, canonicalRequest, ServiceTypeS3) - signingKey := getSigningKey(s.secretAccessKey, s.region, s.reqTime) + signingKey := getSigningKey(s.secretAccessKey, s.region, s.reqTime, ServiceTypeS3) // Calculate signature. s.seedSignature = getSignature(signingKey, stringToSign) @@ -185,7 +185,7 @@ func (s *StreamingReader) signChunk(chunkLen int) { // setStreamingAuthHeader - builds and sets authorization header value // for streaming signature. func (s *StreamingReader) setStreamingAuthHeader(req *http.Request) { - credential := GetCredential(s.accessKeyID, s.region, s.reqTime) + credential := GetCredential(s.accessKeyID, s.region, s.reqTime, ServiceTypeS3) authParts := []string{ signV4Algorithm + " Credential=" + credential, "SignedHeaders=" + getSignedHeaders(*req, ignoredStreamingHeaders), diff --git a/pkg/s3signer/request-signature-streaming_test.go b/pkg/signer/request-signature-streaming_test.go similarity index 99% rename from pkg/s3signer/request-signature-streaming_test.go rename to pkg/signer/request-signature-streaming_test.go index e65061e70..4ee08b4a0 100644 --- a/pkg/s3signer/request-signature-streaming_test.go +++ b/pkg/signer/request-signature-streaming_test.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package s3signer +package signer import ( "bytes" diff --git a/pkg/s3signer/request-signature-v2.go b/pkg/signer/request-signature-v2.go similarity index 99% rename from pkg/s3signer/request-signature-v2.go rename to pkg/signer/request-signature-v2.go index 40ba07130..3179d62b5 100644 --- a/pkg/s3signer/request-signature-v2.go +++ b/pkg/signer/request-signature-v2.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package s3signer +package signer import ( "bytes" diff --git a/pkg/s3signer/request-signature-v2_test.go b/pkg/signer/request-signature-v2_test.go similarity index 98% rename from pkg/s3signer/request-signature-v2_test.go rename to pkg/signer/request-signature-v2_test.go index d94e01252..c73c779b5 100644 --- a/pkg/s3signer/request-signature-v2_test.go +++ b/pkg/signer/request-signature-v2_test.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package s3signer +package signer import ( "sort" diff --git a/pkg/s3signer/request-signature-v4.go b/pkg/signer/request-signature-v4.go similarity index 79% rename from pkg/s3signer/request-signature-v4.go rename to pkg/signer/request-signature-v4.go index ab96b58c5..0f6a4c0a6 100644 --- a/pkg/s3signer/request-signature-v4.go +++ b/pkg/signer/request-signature-v4.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package s3signer +package signer import ( "bytes" @@ -36,6 +36,12 @@ const ( yyyymmdd = "20060102" ) +// Different service types +const ( + ServiceTypeS3 = "s3" + ServiceTypeSTS = "sts" +) + /// /// Excerpts from @lsegal - /// https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258. @@ -47,42 +53,21 @@ const ( /// by other agents) or when customers pass requests through /// proxies, which may modify the user-agent. /// -/// Content-Length: -/// -/// This is ignored from signing because generating a pre-signed -/// URL should not provide a content-length constraint, -/// specifically when vending a S3 pre-signed PUT URL. The -/// corollary to this is that when sending regular requests -/// (non-pre-signed), the signature contains a checksum of the -/// body, which implicitly validates the payload length (since -/// changing the number of bytes would change the checksum) -/// and therefore this header is not valuable in the signature. -/// -/// Content-Type: -/// -/// Signing this header causes quite a number of problems in -/// browser environments, where browsers like to modify and -/// normalize the content-type header in different ways. There is -/// more information on this in https://goo.gl/2E9gyy. Avoiding -/// this field simplifies logic and reduces the possibility of -/// future bugs. /// /// Authorization: /// /// Is skipped for obvious reasons /// var v4IgnoredHeaders = map[string]bool{ - "Authorization": true, - "Content-Type": true, - "Content-Length": true, - "User-Agent": true, + "Authorization": true, + "User-Agent": true, } // getSigningKey hmac seed to calculate final signature. -func getSigningKey(secret, loc string, t time.Time) []byte { +func getSigningKey(secret, loc string, t time.Time, serviceType string) []byte { date := sumHMAC([]byte("AWS4"+secret), []byte(t.Format(yyyymmdd))) location := sumHMAC(date, []byte(loc)) - service := sumHMAC(location, []byte("s3")) + service := sumHMAC(location, []byte(serviceType)) signingKey := sumHMAC(service, []byte("aws4_request")) return signingKey } @@ -94,19 +79,19 @@ func getSignature(signingKey []byte, stringToSign string) string { // getScope generate a string of a specific date, an AWS region, and a // service. -func getScope(location string, t time.Time) string { +func getScope(location string, t time.Time, serviceType string) string { scope := strings.Join([]string{ t.Format(yyyymmdd), location, - "s3", + serviceType, "aws4_request", }, "/") return scope } // GetCredential generate a credential string. -func GetCredential(accessKeyID, location string, t time.Time) string { - scope := getScope(location, t) +func GetCredential(accessKeyID, location string, t time.Time, serviceType string) string { + scope := getScope(location, t, serviceType) return accessKeyID + "/" + scope } @@ -184,7 +169,7 @@ func getSignedHeaders(req http.Request, ignoredHeaders map[string]bool) string { // \n // \n // -func getCanonicalRequest(req http.Request, ignoredHeaders map[string]bool) string { +func getCanonicalRequest(req http.Request, ignoredHeaders map[string]bool, hashedPayload string) string { req.URL.RawQuery = strings.Replace(req.URL.Query().Encode(), "+", "%20", -1) canonicalRequest := strings.Join([]string{ req.Method, @@ -192,15 +177,15 @@ func getCanonicalRequest(req http.Request, ignoredHeaders map[string]bool) strin req.URL.RawQuery, getCanonicalHeaders(req, ignoredHeaders), getSignedHeaders(req, ignoredHeaders), - getHashedPayload(req), + hashedPayload, }, "\n") return canonicalRequest } // getStringToSign a string based on selected query values. -func getStringToSignV4(t time.Time, location, canonicalRequest string) string { +func getStringToSignV4(t time.Time, location, canonicalRequest, serviceType string) string { stringToSign := signV4Algorithm + "\n" + t.Format(iso8601DateFormat) + "\n" - stringToSign = stringToSign + getScope(location, t) + "\n" + stringToSign = stringToSign + getScope(location, t, serviceType) + "\n" stringToSign = stringToSign + hex.EncodeToString(sum256([]byte(canonicalRequest))) return stringToSign } @@ -217,7 +202,7 @@ func PreSignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, loc t := time.Now().UTC() // Get credential string. - credential := GetCredential(accessKeyID, location, t) + credential := GetCredential(accessKeyID, location, t, ServiceTypeS3) // Get all signed headers. signedHeaders := getSignedHeaders(req, v4IgnoredHeaders) @@ -236,13 +221,13 @@ func PreSignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, loc req.URL.RawQuery = query.Encode() // Get canonical request. - canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders) + canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders, getHashedPayload(req)) // Get string to sign from canonical request. - stringToSign := getStringToSignV4(t, location, canonicalRequest) + stringToSign := getStringToSignV4(t, location, canonicalRequest, ServiceTypeS3) // Gext hmac signing key. - signingKey := getSigningKey(secretAccessKey, location, t) + signingKey := getSigningKey(secretAccessKey, location, t, ServiceTypeS3) // Calculate signature. signature := getSignature(signingKey, stringToSign) @@ -257,15 +242,19 @@ func PreSignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, loc // requests. func PostPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, location string) string { // Get signining key. - signingkey := getSigningKey(secretAccessKey, location, t) + signingkey := getSigningKey(secretAccessKey, location, t, ServiceTypeS3) // Calculate signature. signature := getSignature(signingkey, policyBase64) return signature } -// SignV4 sign the request before Do(), in accordance with -// http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html. -func SignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string) *http.Request { +// SignV4STS - signature v4 for STS request. +func SignV4STS(req http.Request, accessKeyID, secretAccessKey, location string) *http.Request { + return signV4(req, accessKeyID, secretAccessKey, "", location, ServiceTypeSTS) +} + +// Internal function called for different service types. +func signV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location, serviceType string) *http.Request { // Signature calculation is not needed for anonymous credentials. if accessKeyID == "" || secretAccessKey == "" { return &req @@ -282,17 +271,25 @@ func SignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, locati req.Header.Set("X-Amz-Security-Token", sessionToken) } + hashedPayload := getHashedPayload(req) + if serviceType == ServiceTypeSTS { + // Content sha256 header is not sent with the request + // but it is expected to have sha256 of payload for signature + // in STS service type request. + req.Header.Del("X-Amz-Content-Sha256") + } + // Get canonical request. - canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders) + canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders, hashedPayload) // Get string to sign from canonical request. - stringToSign := getStringToSignV4(t, location, canonicalRequest) + stringToSign := getStringToSignV4(t, location, canonicalRequest, serviceType) // Get hmac signing key. - signingKey := getSigningKey(secretAccessKey, location, t) + signingKey := getSigningKey(secretAccessKey, location, t, serviceType) // Get credential string. - credential := GetCredential(accessKeyID, location, t) + credential := GetCredential(accessKeyID, location, t, serviceType) // Get all signed headers. signedHeaders := getSignedHeaders(req, v4IgnoredHeaders) @@ -313,3 +310,9 @@ func SignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, locati return &req } + +// SignV4 sign the request before Do(), in accordance with +// http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html. +func SignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string) *http.Request { + return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, ServiceTypeS3) +} diff --git a/pkg/s3signer/request-signature-v4_test.go b/pkg/signer/request-signature-v4_test.go similarity index 99% rename from pkg/s3signer/request-signature-v4_test.go rename to pkg/signer/request-signature-v4_test.go index d0c9e3095..96e33286f 100644 --- a/pkg/s3signer/request-signature-v4_test.go +++ b/pkg/signer/request-signature-v4_test.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package s3signer +package signer import ( "io" diff --git a/pkg/s3signer/request-signature_test.go b/pkg/signer/request-signature_test.go similarity index 99% rename from pkg/s3signer/request-signature_test.go rename to pkg/signer/request-signature_test.go index 3e072cff7..cc14bc4c9 100644 --- a/pkg/s3signer/request-signature_test.go +++ b/pkg/signer/request-signature_test.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package s3signer +package signer import ( "net/http" diff --git a/pkg/s3signer/test-utils_test.go b/pkg/signer/test-utils_test.go similarity index 99% rename from pkg/s3signer/test-utils_test.go rename to pkg/signer/test-utils_test.go index 8a7d8ff28..56ba5727b 100644 --- a/pkg/s3signer/test-utils_test.go +++ b/pkg/signer/test-utils_test.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package s3signer +package signer import ( "bufio" diff --git a/pkg/s3signer/utils.go b/pkg/signer/utils.go similarity index 99% rename from pkg/s3signer/utils.go rename to pkg/signer/utils.go index 934e33a48..2192a3693 100644 --- a/pkg/s3signer/utils.go +++ b/pkg/signer/utils.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package s3signer +package signer import ( "crypto/hmac" diff --git a/pkg/s3signer/utils_test.go b/pkg/signer/utils_test.go similarity index 99% rename from pkg/s3signer/utils_test.go rename to pkg/signer/utils_test.go index 5ec030712..745699ee3 100644 --- a/pkg/s3signer/utils_test.go +++ b/pkg/signer/utils_test.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package s3signer +package signer import ( "fmt" From dea53752321e2b0002f73a469bdc2a8bd8b45729 Mon Sep 17 00:00:00 2001 From: Olivier Cloirec <5033885+clook@users.noreply.github.com> Date: Tue, 14 Apr 2020 23:53:24 +0200 Subject: [PATCH 135/215] Add get bucket versioning (#1266) - Add example for GetBucketVersioning - Add doc for GetBucketVersioning --- api-get-bucket-versioning.go | 78 ++++++++++++++++++++++++++++++ docs/API.md | 43 ++++++++++++++-- examples/s3/getbucketversioning.go | 52 ++++++++++++++++++++ 3 files changed, 169 insertions(+), 4 deletions(-) create mode 100644 api-get-bucket-versioning.go create mode 100644 examples/s3/getbucketversioning.go diff --git a/api-get-bucket-versioning.go b/api-get-bucket-versioning.go new file mode 100644 index 000000000..9b13d6730 --- /dev/null +++ b/api-get-bucket-versioning.go @@ -0,0 +1,78 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "encoding/xml" + "io/ioutil" + "net/http" + "net/url" + + "github.com/minio/minio-go/v6/pkg/s3utils" +) + +// BucketVersioningConfiguration is the versioning configuration structure +type BucketVersioningConfiguration struct { + XMLName xml.Name `xml:"VersioningConfiguration"` + Status string `xml:"Status"` + MfaDelete string `xml:"MfaDelete,omitempty"` +} + +// GetBucketVersioning - get versioning configuration for a bucket. +func (c Client) GetBucketVersioning(bucketName string) (BucketVersioningConfiguration, error) { + return c.GetBucketVersioningWithContext(context.Background(), bucketName) +} + +// GetBucketVersioningWithContext gets the versioning configuration on an existing bucket with a context to control cancellations and timeouts. +func (c Client) GetBucketVersioningWithContext(ctx context.Context, bucketName string) (BucketVersioningConfiguration, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return BucketVersioningConfiguration{}, err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("versioning", "") + + // Execute GET on bucket to get the versioning configuration. + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + }) + + defer closeResponse(resp) + if err != nil { + return BucketVersioningConfiguration{}, err + } + + if resp.StatusCode != http.StatusOK { + return BucketVersioningConfiguration{}, httpRespToErrorResponse(resp, bucketName, "") + } + + bucketVersioningBuf, err := ioutil.ReadAll(resp.Body) + if err != nil { + return BucketVersioningConfiguration{}, err + } + + versioningConfig := BucketVersioningConfiguration{} + if err := xml.Unmarshal(bucketVersioningBuf, &versioningConfig); err != nil { + return BucketVersioningConfiguration{}, err + } + return versioningConfig, nil +} diff --git a/docs/API.md b/docs/API.md index 9e4403065..cac3b9a49 100644 --- a/docs/API.md +++ b/docs/API.md @@ -64,10 +64,10 @@ func main() { | | [`ComposeObject`](#ComposeObject) | [`ComposeObject`](#ComposeObject) | | [`GetObjectLockConfig`](#GetObjectLockConfig) | | | | [`NewSourceInfo`](#NewSourceInfo) | [`NewSourceInfo`](#NewSourceInfo) | | [`EnableVersioning`](#EnableVersioning) | | | | [`NewDestinationInfo`](#NewDestinationInfo) | [`NewDestinationInfo`](#NewDestinationInfo) | | [`DisableVersioning`](#DisableVersioning) | | -| | [`PutObjectWithContext`](#PutObjectWithContext) | [`PutObjectWithContext`](#PutObjectWithContext) | | [`SetBucketEncryption`](#SetBucketEncryption) | -| | [`GetObjectWithContext`](#GetObjectWithContext) | [`GetObjectWithContext`](#GetObjectWithContext) | | [`GetBucketEncryption`](#GetBucketEncryption) | | -| | [`FPutObjectWithContext`](#FPutObjectWithContext) | [`FPutObjectWithContext`](#FPutObjectWithContext) | | [`DeleteBucketEncryption`](#DeleteBucketEncryption) | -| | [`FGetObjectWithContext`](#FGetObjectWithContext) | [`FGetObjectWithContext`](#FGetObjectWithContext) | | | +| | [`PutObjectWithContext`](#PutObjectWithContext) | [`PutObjectWithContext`](#PutObjectWithContext) | | [`GetBucketVersioning`](#GetBucketVersioning) | +| | [`GetObjectWithContext`](#GetObjectWithContext) | [`GetObjectWithContext`](#GetObjectWithContext) | | [`SetBucketEncryption`](#SetBucketEncryption) | | +| | [`FPutObjectWithContext`](#FPutObjectWithContext) | [`FPutObjectWithContext`](#FPutObjectWithContext) | | [`GetBucketEncryption`](#GetBucketEncryption) | +| | [`FGetObjectWithContext`](#FGetObjectWithContext) | [`FGetObjectWithContext`](#FGetObjectWithContext) | | [`DeleteBucketEncryption`](#DeleteBucketEncryption) | | | [`RemoveObjectsWithContext`](#RemoveObjectsWithContext) | | | | | | [`RemoveObjectsWithOptions`](#RemoveObjectsWithOptions) | | | | | | [`RemoveObjectsWithOptionsContext`](#RemoveObjectsWithOptionsContext) | | | | @@ -2314,6 +2314,41 @@ if err != nil { fmt.Println("versioning disabled for bucket 'my-bucketname'") ``` + +### GetBucketVersioning(bucketName string) (BucketVersioningConfiguration, error) +Get versioning configuration set on a bucket. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | + +__Return Values__ + + +|Param |Type |Description | +|:---|:---| :---| +|`configuration` | _minio.BucketVersioningConfiguration_ | Structure that holds versioning configuration | +|`err` | _error_ |Standard Error | + +__Example__ + +```go +s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) +if err != nil { + log.Fatalln(err) +} + +// Get versioning configuration set on an S3 bucket and print it out +versioningConfig, err := s3Client.GetBucketVersioning("my-bucketname") +if err != nil { + log.Fatalln(err) +} +fmt.Printf("%+v\n", versioningConfig) +``` + ## 7. Client custom settings diff --git a/examples/s3/getbucketversioning.go b/examples/s3/getbucketversioning.go new file mode 100644 index 000000000..daf89cb96 --- /dev/null +++ b/examples/s3/getbucketversioning.go @@ -0,0 +1,52 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + "log" + + "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // s3Client.TraceOn(os.Stderr) + + // Get versioning configuration set on an S3 bucket, + // and print out the versioning configuration. + versioningConfig, err := s3Client.GetBucketVersioning("my-bucketname") + if err != nil { + log.Fatalln(err) + } + fmt.Printf("%+v\n", versioningConfig) +} From 16fa71b209e69ba726350bfc24d70eb233906ccc Mon Sep 17 00:00:00 2001 From: Chanho Yong Date: Wed, 15 Apr 2020 07:38:42 +0900 Subject: [PATCH 136/215] fix: AWS STS url to use https for web-identity (#1263) --- pkg/credentials/iam_aws.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/credentials/iam_aws.go b/pkg/credentials/iam_aws.go index 1b67cbfad..60f78f808 100644 --- a/pkg/credentials/iam_aws.go +++ b/pkg/credentials/iam_aws.go @@ -82,7 +82,7 @@ func (m *IAM) Retrieve() (Value, error) { case len(os.Getenv("AWS_WEB_IDENTITY_TOKEN_FILE")) > 0: if len(endpoint) == 0 { if len(os.Getenv("AWS_REGION")) > 0 { - endpoint = "sts." + os.Getenv("AWS_REGION") + ".amazonaws.com" + endpoint = "https://sts." + os.Getenv("AWS_REGION") + ".amazonaws.com" } else { endpoint = defaultSTSRoleEndpoint } From 337bb00bc3c832292b36681c7bde1b56a185c310 Mon Sep 17 00:00:00 2001 From: Daniel Valdivia Date: Tue, 14 Apr 2020 16:25:54 -0700 Subject: [PATCH 137/215] return an error when removing an event matches nothing (#1267) --- bucket-notification.go | 29 +- bucket-notification_test.go | 641 +++++++++++++++++++++++++++++++++++- 2 files changed, 651 insertions(+), 19 deletions(-) diff --git a/bucket-notification.go b/bucket-notification.go index 277c2e11d..7503b01b9 100644 --- a/bucket-notification.go +++ b/bucket-notification.go @@ -19,7 +19,9 @@ package minio import ( "encoding/xml" + "errors" "fmt" + "github.com/minio/minio-go/v6/pkg/set" ) @@ -175,11 +177,7 @@ func EqualFilterRuleList(a, b []FilterRule) bool { } // Equal returns whether this `NotificationConfig` is equal to another defined by the passed parameters -func (t *NotificationConfig) Equal(arn string, events []NotificationEventType, prefix, suffix string) bool { - // if it's not the same ARN, return immediately. - if t.Arn.String() != arn { - return false - } +func (t *NotificationConfig) Equal(events []NotificationEventType, prefix, suffix string) bool { //Compare events passEvents := EqualNotificationEventTypeList(t.Events, events) @@ -310,19 +308,24 @@ func (b *BucketNotification) RemoveTopicByArn(arn Arn) { b.TopicConfigs = topics } +// ErrNoNotificationConfigMatch is returned when a notification configuration (sqs,sns,lambda) is not found when trying to delete +var ErrNoNotificationConfigMatch = errors.New("no notification configuration matched") + // RemoveTopicByArnEventsPrefixSuffix removes a topic configuration that match the exact specified ARN, events, prefix and suffix -func (b *BucketNotification) RemoveTopicByArnEventsPrefixSuffix(arn Arn, events []NotificationEventType, prefix, suffix string) { +func (b *BucketNotification) RemoveTopicByArnEventsPrefixSuffix(arn Arn, events []NotificationEventType, prefix, suffix string) error { removeIndex := -1 for i, v := range b.TopicConfigs { // if it matches events and filters, mark the index for deletion - if v.NotificationConfig.Equal(arn.String(), events, prefix, suffix) { + if v.Topic == arn.String() && v.NotificationConfig.Equal(events, prefix, suffix) { removeIndex = i break // since we have at most one matching config } } if removeIndex >= 0 { b.TopicConfigs = append(b.TopicConfigs[:removeIndex], b.TopicConfigs[removeIndex+1:]...) + return nil } + return ErrNoNotificationConfigMatch } // RemoveQueueByArn removes all queue configurations that match the exact specified ARN @@ -337,18 +340,20 @@ func (b *BucketNotification) RemoveQueueByArn(arn Arn) { } // RemoveQueueByArnEventsPrefixSuffix removes a queue configuration that match the exact specified ARN, events, prefix and suffix -func (b *BucketNotification) RemoveQueueByArnEventsPrefixSuffix(arn Arn, events []NotificationEventType, prefix, suffix string) { +func (b *BucketNotification) RemoveQueueByArnEventsPrefixSuffix(arn Arn, events []NotificationEventType, prefix, suffix string) error { removeIndex := -1 for i, v := range b.QueueConfigs { // if it matches events and filters, mark the index for deletion - if v.NotificationConfig.Equal(arn.String(), events, prefix, suffix) { + if v.Queue == arn.String() && v.NotificationConfig.Equal(events, prefix, suffix) { removeIndex = i break // since we have at most one matching config } } if removeIndex >= 0 { b.QueueConfigs = append(b.QueueConfigs[:removeIndex], b.QueueConfigs[removeIndex+1:]...) + return nil } + return ErrNoNotificationConfigMatch } // RemoveLambdaByArn removes all lambda configurations that match the exact specified ARN @@ -363,16 +368,18 @@ func (b *BucketNotification) RemoveLambdaByArn(arn Arn) { } // RemoveLambdaByArnEventsPrefixSuffix removes a topic configuration that match the exact specified ARN, events, prefix and suffix -func (b *BucketNotification) RemoveLambdaByArnEventsPrefixSuffix(arn Arn, events []NotificationEventType, prefix, suffix string) { +func (b *BucketNotification) RemoveLambdaByArnEventsPrefixSuffix(arn Arn, events []NotificationEventType, prefix, suffix string) error { removeIndex := -1 for i, v := range b.LambdaConfigs { // if it matches events and filters, mark the index for deletion - if v.NotificationConfig.Equal(arn.String(), events, prefix, suffix) { + if v.Lambda == arn.String() && v.NotificationConfig.Equal(events, prefix, suffix) { removeIndex = i break // since we have at most one matching config } } if removeIndex >= 0 { b.LambdaConfigs = append(b.LambdaConfigs[:removeIndex], b.LambdaConfigs[removeIndex+1:]...) + return nil } + return ErrNoNotificationConfigMatch } diff --git a/bucket-notification_test.go b/bucket-notification_test.go index a0c5ef30f..74d43352e 100644 --- a/bucket-notification_test.go +++ b/bucket-notification_test.go @@ -17,7 +17,10 @@ package minio -import "testing" +import ( + "encoding/xml" + "testing" +) func TestEqualNotificationEventTypeList(t *testing.T) { type args struct { @@ -113,7 +116,6 @@ func TestNotificationConfig_Equal(t *testing.T) { Filter *Filter } type args struct { - arn string events []NotificationEventType prefix string suffix string @@ -136,7 +138,6 @@ func TestNotificationConfig_Equal(t *testing.T) { }, }, args: args{ - arn: "arn:minio:sqs::1:postgresql", events: []NotificationEventType{ObjectCreatedAll, ObjectAccessedAll}, prefix: "prefix1", suffix: "suffix1", @@ -155,7 +156,6 @@ func TestNotificationConfig_Equal(t *testing.T) { }, }, args: args{ - arn: "arn:minio:sqs::1:postgresql", events: []NotificationEventType{ObjectCreatedAll, ObjectAccessedAll}, prefix: "prefix1", suffix: "suffix1", @@ -174,7 +174,6 @@ func TestNotificationConfig_Equal(t *testing.T) { }, }, args: args{ - arn: "arn:minio:sqs::1:postgresql", events: []NotificationEventType{ObjectCreatedAll, ObjectAccessedAll}, prefix: "prefix1", suffix: "suffix1", @@ -184,7 +183,6 @@ func TestNotificationConfig_Equal(t *testing.T) { { name: "different arn", fields: fields{ - Arn: NewArn("minio", "sqs", "", "2", "postgresql"), Events: []NotificationEventType{ObjectAccessedAll}, Filter: &Filter{ S3Key: S3Key{ @@ -193,7 +191,6 @@ func TestNotificationConfig_Equal(t *testing.T) { }, }, args: args{ - arn: "arn:minio:sqs::1:postgresql", events: []NotificationEventType{ObjectCreatedAll, ObjectAccessedAll}, prefix: "prefix1", suffix: "suffix1", @@ -209,9 +206,637 @@ func TestNotificationConfig_Equal(t *testing.T) { Events: tt.fields.Events, Filter: tt.fields.Filter, } - if got := nc.Equal(tt.args.arn, tt.args.events, tt.args.prefix, tt.args.suffix); got != tt.want { + if got := nc.Equal(tt.args.events, tt.args.prefix, tt.args.suffix); got != tt.want { t.Errorf("Equal() = %v, want %v", got, tt.want) } }) } } + +func TestBucketNotification_RemoveQueueByArnEventsPrefixSuffix(t *testing.T) { + type fields struct { + XMLName xml.Name + LambdaConfigs []LambdaConfig + TopicConfigs []TopicConfig + QueueConfigs []QueueConfig + } + type args struct { + arn Arn + events []NotificationEventType + prefix string + suffix string + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "Queue Configuration Removed with events, prefix", + fields: fields{ + XMLName: xml.Name{}, + LambdaConfigs: nil, + TopicConfigs: nil, + QueueConfigs: []QueueConfig{ + { + NotificationConfig: NotificationConfig{ + ID: "", + Arn: Arn{ + Partition: "minio", + Service: "sqs", + Region: "", + AccountID: "1", + Resource: "postgresql", + }, + Events: []NotificationEventType{ + ObjectAccessedAll, + }, + Filter: &Filter{ + S3Key: S3Key{ + FilterRules: []FilterRule{ + { + Name: "prefix", + Value: "x", + }, + }, + }, + }, + }, + Queue: "arn:minio:sqs::1:postgresql", + }, + }, + }, + args: args{ + arn: Arn{ + Partition: "minio", + Service: "sqs", + Region: "", + AccountID: "1", + Resource: "postgresql", + }, + events: []NotificationEventType{ + ObjectAccessedAll, + }, + prefix: "x", + suffix: "", + }, + wantErr: false, + }, + { + name: "Queue Configuration Removed with events, prefix, suffix", + fields: fields{ + XMLName: xml.Name{}, + LambdaConfigs: nil, + TopicConfigs: nil, + QueueConfigs: []QueueConfig{ + { + NotificationConfig: NotificationConfig{ + ID: "", + Arn: Arn{ + Partition: "minio", + Service: "sqs", + Region: "", + AccountID: "1", + Resource: "postgresql", + }, + Events: []NotificationEventType{ + ObjectAccessedAll, + }, + Filter: &Filter{ + S3Key: S3Key{ + FilterRules: []FilterRule{ + { + Name: "prefix", + Value: "x", + }, + { + Name: "suffix", + Value: "y", + }, + }, + }, + }, + }, + Queue: "arn:minio:sqs::1:postgresql", + }, + }, + }, + args: args{ + arn: Arn{ + Partition: "minio", + Service: "sqs", + Region: "", + AccountID: "1", + Resource: "postgresql", + }, + events: []NotificationEventType{ + ObjectAccessedAll, + }, + prefix: "x", + suffix: "y", + }, + wantErr: false, + }, + { + name: "Error Returned Queue Configuration Not Removed", + fields: fields{ + XMLName: xml.Name{}, + LambdaConfigs: nil, + TopicConfigs: nil, + QueueConfigs: []QueueConfig{ + { + NotificationConfig: NotificationConfig{ + ID: "", + Arn: Arn{ + Partition: "minio", + Service: "sqs", + Region: "", + AccountID: "1", + Resource: "postgresql", + }, + Events: []NotificationEventType{ + ObjectAccessedAll, + }, + Filter: &Filter{ + S3Key: S3Key{ + FilterRules: []FilterRule{ + { + Name: "prefix", + Value: "x", + }, + }, + }, + }, + }, + Queue: "arn:minio:sqs::1:postgresql", + }, + }, + }, + args: args{ + arn: Arn{ + Partition: "minio", + Service: "sqs", + Region: "", + AccountID: "1", + Resource: "postgresql", + }, + events: []NotificationEventType{ + ObjectAccessedAll, + }, + prefix: "", + suffix: "", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := &BucketNotification{ + XMLName: tt.fields.XMLName, + LambdaConfigs: tt.fields.LambdaConfigs, + TopicConfigs: tt.fields.TopicConfigs, + QueueConfigs: tt.fields.QueueConfigs, + } + if err := b.RemoveQueueByArnEventsPrefixSuffix(tt.args.arn, tt.args.events, tt.args.prefix, tt.args.suffix); (err != nil) != tt.wantErr { + t.Errorf("RemoveQueueByArnEventsPrefixSuffix() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestBucketNotification_RemoveLambdaByArnEventsPrefixSuffix(t *testing.T) { + type fields struct { + XMLName xml.Name + LambdaConfigs []LambdaConfig + TopicConfigs []TopicConfig + QueueConfigs []QueueConfig + } + type args struct { + arn Arn + events []NotificationEventType + prefix string + suffix string + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "Lambda Configuration Removed with events, prefix", + fields: fields{ + XMLName: xml.Name{}, + QueueConfigs: nil, + TopicConfigs: nil, + LambdaConfigs: []LambdaConfig{ + { + NotificationConfig: NotificationConfig{ + ID: "", + Arn: Arn{ + Partition: "minio", + Service: "lambda", + Region: "", + AccountID: "1", + Resource: "provider", + }, + Events: []NotificationEventType{ + ObjectAccessedAll, + }, + Filter: &Filter{ + S3Key: S3Key{ + FilterRules: []FilterRule{ + { + Name: "prefix", + Value: "x", + }, + }, + }, + }, + }, + Lambda: "arn:minio:lambda::1:provider", + }, + }, + }, + args: args{ + arn: Arn{ + Partition: "minio", + Service: "lambda", + Region: "", + AccountID: "1", + Resource: "provider", + }, + events: []NotificationEventType{ + ObjectAccessedAll, + }, + prefix: "x", + suffix: "", + }, + wantErr: false, + }, + { + name: "Lambda Configuration Removed with events, prefix, suffix", + fields: fields{ + XMLName: xml.Name{}, + QueueConfigs: nil, + TopicConfigs: nil, + LambdaConfigs: []LambdaConfig{ + { + NotificationConfig: NotificationConfig{ + ID: "", + Arn: Arn{ + Partition: "minio", + Service: "lambda", + Region: "", + AccountID: "1", + Resource: "provider", + }, + Events: []NotificationEventType{ + ObjectAccessedAll, + }, + Filter: &Filter{ + S3Key: S3Key{ + FilterRules: []FilterRule{ + { + Name: "prefix", + Value: "x", + }, + { + Name: "suffix", + Value: "y", + }, + }, + }, + }, + }, + Lambda: "arn:minio:lambda::1:provider", + }, + }, + }, + args: args{ + arn: Arn{ + Partition: "minio", + Service: "lambda", + Region: "", + AccountID: "1", + Resource: "provider", + }, + events: []NotificationEventType{ + ObjectAccessedAll, + }, + prefix: "x", + suffix: "y", + }, + wantErr: false, + }, + { + name: "Error Returned Lambda Configuration Not Removed", + fields: fields{ + XMLName: xml.Name{}, + QueueConfigs: nil, + TopicConfigs: nil, + LambdaConfigs: []LambdaConfig{ + { + NotificationConfig: NotificationConfig{ + ID: "", + Arn: Arn{ + Partition: "minio", + Service: "lambda", + Region: "", + AccountID: "1", + Resource: "provider", + }, + Events: []NotificationEventType{ + ObjectAccessedAll, + }, + Filter: &Filter{ + S3Key: S3Key{ + FilterRules: []FilterRule{ + { + Name: "prefix", + Value: "x", + }, + }, + }, + }, + }, + Lambda: "arn:minio:lambda::1:provider", + }, + }, + }, + args: args{ + arn: Arn{ + Partition: "minio", + Service: "lambda", + Region: "", + AccountID: "1", + Resource: "provider", + }, + events: []NotificationEventType{ + ObjectAccessedAll, + }, + prefix: "", + suffix: "", + }, + wantErr: true, + }, + { + name: "Error Returned Invalid ARN", + fields: fields{ + XMLName: xml.Name{}, + QueueConfigs: nil, + TopicConfigs: nil, + LambdaConfigs: []LambdaConfig{ + { + NotificationConfig: NotificationConfig{ + ID: "", + Arn: Arn{ + Partition: "minio", + Service: "lambda", + Region: "", + AccountID: "1", + Resource: "provider", + }, + Events: []NotificationEventType{ + ObjectAccessedAll, + }, + Filter: &Filter{ + S3Key: S3Key{ + FilterRules: []FilterRule{ + { + Name: "prefix", + Value: "x", + }, + }, + }, + }, + }, + Lambda: "arn:minio:lambda::1:provider", + }, + }, + }, + args: args{ + arn: Arn{ + Partition: "minio", + + Service: "lambda", + Region: "", + AccountID: "2", + Resource: "provider", + }, + events: []NotificationEventType{ + ObjectAccessedAll, + }, + prefix: "", + suffix: "", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := &BucketNotification{ + XMLName: tt.fields.XMLName, + LambdaConfigs: tt.fields.LambdaConfigs, + TopicConfigs: tt.fields.TopicConfigs, + QueueConfigs: tt.fields.QueueConfigs, + } + if err := b.RemoveLambdaByArnEventsPrefixSuffix(tt.args.arn, tt.args.events, tt.args.prefix, tt.args.suffix); (err != nil) != tt.wantErr { + t.Errorf("RemoveLambdaByArnEventsPrefixSuffix() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestBucketNotification_RemoveTopicByArnEventsPrefixSuffix(t *testing.T) { + type fields struct { + XMLName xml.Name + LambdaConfigs []LambdaConfig + TopicConfigs []TopicConfig + QueueConfigs []QueueConfig + } + type args struct { + arn Arn + events []NotificationEventType + prefix string + suffix string + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "Topic Configuration Removed with events, prefix", + fields: fields{ + XMLName: xml.Name{}, + QueueConfigs: nil, + LambdaConfigs: nil, + TopicConfigs: []TopicConfig{ + { + NotificationConfig: NotificationConfig{ + ID: "", + Arn: Arn{ + Partition: "minio", + Service: "sns", + Region: "", + AccountID: "1", + Resource: "kafka", + }, + Events: []NotificationEventType{ + ObjectAccessedAll, + }, + Filter: &Filter{ + S3Key: S3Key{ + FilterRules: []FilterRule{ + { + Name: "prefix", + Value: "x", + }, + }, + }, + }, + }, + Topic: "arn:minio:sns::1:kafka", + }, + }, + }, + args: args{ + arn: Arn{ + Partition: "minio", + Service: "sns", + Region: "", + AccountID: "1", + Resource: "kafka", + }, + events: []NotificationEventType{ + ObjectAccessedAll, + }, + prefix: "x", + suffix: "", + }, + wantErr: false, + }, + { + name: "Topic Configuration Removed with events, prefix, suffix", + fields: fields{ + XMLName: xml.Name{}, + QueueConfigs: nil, + LambdaConfigs: nil, + TopicConfigs: []TopicConfig{ + { + NotificationConfig: NotificationConfig{ + ID: "", + Arn: Arn{ + Partition: "minio", + Service: "sns", + Region: "", + AccountID: "1", + Resource: "kafka", + }, + Events: []NotificationEventType{ + ObjectAccessedAll, + }, + Filter: &Filter{ + S3Key: S3Key{ + FilterRules: []FilterRule{ + { + Name: "prefix", + Value: "x", + }, + { + Name: "suffix", + Value: "y", + }, + }, + }, + }, + }, + Topic: "arn:minio:sns::1:kafka", + }, + }, + }, + args: args{ + arn: Arn{ + Partition: "minio", + Service: "sns", + Region: "", + AccountID: "1", + Resource: "kafka", + }, + events: []NotificationEventType{ + ObjectAccessedAll, + }, + prefix: "x", + suffix: "y", + }, + wantErr: false, + }, + { + name: "Error Returned Topic Configuration Not Removed", + fields: fields{ + XMLName: xml.Name{}, + QueueConfigs: nil, + LambdaConfigs: nil, + TopicConfigs: []TopicConfig{ + { + NotificationConfig: NotificationConfig{ + ID: "", + Arn: Arn{ + Partition: "minio", + Service: "sns", + Region: "", + AccountID: "1", + Resource: "kafka", + }, + Events: []NotificationEventType{ + ObjectAccessedAll, + }, + Filter: &Filter{ + S3Key: S3Key{ + FilterRules: []FilterRule{ + { + Name: "prefix", + Value: "x", + }, + }, + }, + }, + }, + Topic: "arn:minio:sns::1:kafka", + }, + }, + }, + args: args{ + arn: Arn{ + Partition: "minio", + Service: "sns", + Region: "", + AccountID: "1", + Resource: "kafka", + }, + events: []NotificationEventType{ + ObjectAccessedAll, + }, + prefix: "", + suffix: "", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := &BucketNotification{ + XMLName: tt.fields.XMLName, + LambdaConfigs: tt.fields.LambdaConfigs, + TopicConfigs: tt.fields.TopicConfigs, + QueueConfigs: tt.fields.QueueConfigs, + } + if err := b.RemoveTopicByArnEventsPrefixSuffix(tt.args.arn, tt.args.events, tt.args.prefix, tt.args.suffix); (err != nil) != tt.wantErr { + t.Errorf("RemoveTopicByArnEventsPrefixSuffix() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} From fc08315fd86aa04b8fddc5d2855621603b700581 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Tue, 14 Apr 2020 23:34:38 +0000 Subject: [PATCH 138/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index cb094ea32..9ec0a6363 100644 --- a/api.go +++ b/api.go @@ -104,7 +104,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.53" + libraryVersion = "v6.0.54" ) // User Agent should always following the below style. From d624b8a802c57977f852715a31b3324f95efc9a2 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 17 Apr 2020 00:23:47 -0700 Subject: [PATCH 139/215] preserve data after calculating md5sum (#1270) --- api-put-object-streaming.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/api-put-object-streaming.go b/api-put-object-streaming.go index 9463062a1..4da4174cd 100644 --- a/api-put-object-streaming.go +++ b/api-put-object-streaming.go @@ -363,14 +363,21 @@ func (c Client) putObject(ctx context.Context, bucketName, objectName string, re if size < 0 && !s3utils.IsGoogleEndpoint(*c.endpointURL) { return 0, ErrEntityTooSmall(size, bucketName, objectName) } + + if opts.SendContentMd5 && s3utils.IsGoogleEndpoint(*c.endpointURL) && size < 0 { + return 0, ErrInvalidArgument("MD5Sum cannot be calculated with size '-1'") + } + if size > 0 { if isReadAt(reader) && !isObject(reader) { - seeker, _ := reader.(io.Seeker) - offset, err := seeker.Seek(0, io.SeekCurrent) - if err != nil { - return 0, ErrInvalidArgument(err.Error()) + seeker, ok := reader.(io.Seeker) + if ok { + offset, err := seeker.Seek(0, io.SeekCurrent) + if err != nil { + return 0, ErrInvalidArgument(err.Error()) + } + reader = io.NewSectionReader(reader.(io.ReaderAt), offset, size) } - reader = io.NewSectionReader(reader.(io.ReaderAt), offset, size) } } @@ -389,6 +396,7 @@ func (c Client) putObject(ctx context.Context, bucketName, objectName string, re hash := md5.New() hash.Write(buf[:length]) md5Base64 = base64.StdEncoding.EncodeToString(hash.Sum(nil)) + reader = bytes.NewReader(buf[:length]) } // Update progress reader appropriately to the latest offset as we From 33e5528a518671ee5595cbd180382e9286520d53 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 17 Apr 2020 16:40:46 -0700 Subject: [PATCH 140/215] fix increase default BufferSize for bufio.Scanner (#1272) Use a higher buffer to handle proxy caching issues might occur. --- api-notification.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api-notification.go b/api-notification.go index b08ad993a..57ac1ed37 100644 --- a/api-notification.go +++ b/api-notification.go @@ -138,6 +138,8 @@ type NotificationInfo struct { // ListenBucketNotification - listen on bucket notifications. func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, events []string, doneCh <-chan struct{}) <-chan NotificationInfo { notificationInfoCh := make(chan NotificationInfo, 1) + const notificationCapacity = 1024 * 1024 + notificationEventBuffer := make([]byte, notificationCapacity) // Only success, start a routine to start reading line by line. go func(notificationInfoCh chan<- NotificationInfo) { defer close(notificationInfoCh) @@ -198,6 +200,10 @@ func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, even // Initialize a new bufio scanner, to read line by line. bio := bufio.NewScanner(resp.Body) + // Use a higher buffer to support unexpected + // caching done by proxies + bio.Buffer(notificationEventBuffer, notificationCapacity) + // Unmarshal each line, returns marshalled values. for bio.Scan() { var notificationInfo NotificationInfo From 22537a49993edea18d5e38dc1fcdb019a8c9bf64 Mon Sep 17 00:00:00 2001 From: Anis Elleuch Date: Sat, 18 Apr 2020 17:27:19 +0100 Subject: [PATCH 141/215] sql: Add SQL API to set CSV/JSON options (#1268) In S3 SQL, there is a difference between a *) non-set parameter *) a parameter set to empty *) a parameter set to a non empty value For that, users need to explicity call a method to set an option, if not done, the default value in server's side will be used. --- api-select.go | 241 +++++++++++++++++++++++++++++++++++++++++++++++--- appveyor.yml | 2 +- 2 files changed, 228 insertions(+), 15 deletions(-) diff --git a/api-select.go b/api-select.go index 7ecd95380..f9f4dda58 100644 --- a/api-select.go +++ b/api-select.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * (C) 2018 MinIO, Inc. + * (C) 2018-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -88,31 +88,244 @@ type ParquetInputOptions struct{} // CSVInputOptions csv input specific options type CSVInputOptions struct { - FileHeaderInfo CSVFileHeaderInfo - RecordDelimiter string - FieldDelimiter string `xml:",omitempty"` - QuoteCharacter string `xml:",omitempty"` - QuoteEscapeCharacter string `xml:",omitempty"` - Comments string `xml:",omitempty"` + FileHeaderInfo CSVFileHeaderInfo + fileHeaderInfoSet bool + + RecordDelimiter string + recordDelimiterSet bool + + FieldDelimiter string + fieldDelimiterSet bool + + QuoteCharacter string + quoteCharacterSet bool + + QuoteEscapeCharacter string + quoteEscapeCharacterSet bool + + Comments string + commentsSet bool +} + +// SetFileHeaderInfo sets the file header info in the CSV input options +func (c *CSVInputOptions) SetFileHeaderInfo(val CSVFileHeaderInfo) { + c.FileHeaderInfo = val + c.fileHeaderInfoSet = true +} + +// SetRecordDelimiter sets the record delimiter in the CSV input options +func (c *CSVInputOptions) SetRecordDelimiter(val string) { + c.RecordDelimiter = val + c.recordDelimiterSet = true +} + +// SetFieldDelimiter sets the field delimiter in the CSV input options +func (c *CSVInputOptions) SetFieldDelimiter(val string) { + c.FieldDelimiter = val + c.fieldDelimiterSet = true +} + +// SetQuoteCharacter sets the quote character in the CSV input options +func (c *CSVInputOptions) SetQuoteCharacter(val string) { + c.QuoteCharacter = val + c.quoteCharacterSet = true +} + +// SetQuoteEscapeCharacter sets the quote escape character in the CSV input options +func (c *CSVInputOptions) SetQuoteEscapeCharacter(val string) { + c.QuoteEscapeCharacter = val + c.quoteEscapeCharacterSet = true +} + +// SetComments sets the comments character in the CSV input options +func (c *CSVInputOptions) SetComments(val string) { + c.Comments = val + c.commentsSet = true +} + +// MarshalXML - produces the xml representation of the CSV input options struct +func (c CSVInputOptions) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if err := e.EncodeToken(start); err != nil { + return err + } + if c.FileHeaderInfo != "" || c.fileHeaderInfoSet { + if err := e.EncodeElement(c.FileHeaderInfo, xml.StartElement{Name: xml.Name{Local: "FileHeaderInfo"}}); err != nil { + return err + } + } + + if c.RecordDelimiter != "" || c.recordDelimiterSet { + if err := e.EncodeElement(c.RecordDelimiter, xml.StartElement{Name: xml.Name{Local: "RecordDelimiter"}}); err != nil { + return err + } + } + + if c.FieldDelimiter != "" || c.fieldDelimiterSet { + if err := e.EncodeElement(c.FieldDelimiter, xml.StartElement{Name: xml.Name{Local: "FieldDelimiter"}}); err != nil { + return err + } + } + + if c.QuoteCharacter != "" || c.quoteCharacterSet { + if err := e.EncodeElement(c.QuoteCharacter, xml.StartElement{Name: xml.Name{Local: "QuoteCharacter"}}); err != nil { + return err + } + } + + if c.QuoteEscapeCharacter != "" || c.quoteEscapeCharacterSet { + if err := e.EncodeElement(c.QuoteEscapeCharacter, xml.StartElement{Name: xml.Name{Local: "QuoteEscapeCharacter"}}); err != nil { + return err + } + } + + if c.Comments != "" || c.commentsSet { + if err := e.EncodeElement(c.Comments, xml.StartElement{Name: xml.Name{Local: "Comments"}}); err != nil { + return err + } + } + + return e.EncodeToken(xml.EndElement{Name: start.Name}) } // CSVOutputOptions csv output specific options type CSVOutputOptions struct { - QuoteFields CSVQuoteFields `xml:",omitempty"` - RecordDelimiter string - FieldDelimiter string `xml:",omitempty"` - QuoteCharacter string `xml:",omitempty"` - QuoteEscapeCharacter string `xml:",omitempty"` + QuoteFields CSVQuoteFields + quoteFieldsSet bool + + RecordDelimiter string + recordDelimiterSet bool + + FieldDelimiter string + fieldDelimiterSet bool + + QuoteCharacter string + quoteCharacterSet bool + + QuoteEscapeCharacter string + quoteEscapeCharacterSet bool +} + +// SetQuoteFields sets the quote field parameter in the CSV output options +func (c *CSVOutputOptions) SetQuoteFields(val CSVQuoteFields) { + c.QuoteFields = val + c.quoteFieldsSet = true +} + +// SetRecordDelimiter sets the record delimiter character in the CSV output options +func (c *CSVOutputOptions) SetRecordDelimiter(val string) { + c.RecordDelimiter = val + c.recordDelimiterSet = true +} + +// SetFieldDelimiter sets the field delimiter character in the CSV output options +func (c *CSVOutputOptions) SetFieldDelimiter(val string) { + c.FieldDelimiter = val + c.fieldDelimiterSet = true +} + +// SetQuoteCharacter sets the quote character in the CSV output options +func (c *CSVOutputOptions) SetQuoteCharacter(val string) { + c.QuoteCharacter = val + c.quoteCharacterSet = true +} + +// SetQuoteEscapeCharacter sets the quote escape character in the CSV output options +func (c *CSVOutputOptions) SetQuoteEscapeCharacter(val string) { + c.QuoteEscapeCharacter = val + c.quoteEscapeCharacterSet = true +} + +// MarshalXML - produces the xml representation of the CSVOutputOptions struct +func (c CSVOutputOptions) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if err := e.EncodeToken(start); err != nil { + return err + } + + if c.QuoteFields != "" || c.quoteFieldsSet { + if err := e.EncodeElement(c.QuoteFields, xml.StartElement{Name: xml.Name{Local: "QuoteFields"}}); err != nil { + return err + } + } + + if c.RecordDelimiter != "" || c.recordDelimiterSet { + if err := e.EncodeElement(c.RecordDelimiter, xml.StartElement{Name: xml.Name{Local: "RecordDelimiter"}}); err != nil { + return err + } + } + + if c.FieldDelimiter != "" || c.fieldDelimiterSet { + if err := e.EncodeElement(c.FieldDelimiter, xml.StartElement{Name: xml.Name{Local: "FieldDelimiter"}}); err != nil { + return err + } + } + + if c.QuoteCharacter != "" || c.quoteCharacterSet { + if err := e.EncodeElement(c.QuoteCharacter, xml.StartElement{Name: xml.Name{Local: "QuoteCharacter"}}); err != nil { + return err + } + } + + if c.QuoteEscapeCharacter != "" || c.quoteEscapeCharacterSet { + if err := e.EncodeElement(c.QuoteEscapeCharacter, xml.StartElement{Name: xml.Name{Local: "QuoteEscapeCharacter"}}); err != nil { + return err + } + } + + return e.EncodeToken(xml.EndElement{Name: start.Name}) } // JSONInputOptions json input specific options type JSONInputOptions struct { - Type JSONType + Type JSONType + typeSet bool +} + +// SetType sets the JSON type in the JSON input options +func (j *JSONInputOptions) SetType(typ JSONType) { + j.Type = typ + j.typeSet = true +} + +// MarshalXML - produces the xml representation of the JSONInputOptions struct +func (j JSONInputOptions) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if err := e.EncodeToken(start); err != nil { + return err + } + + if j.Type != "" || j.typeSet { + if err := e.EncodeElement(j.Type, xml.StartElement{Name: xml.Name{Local: "Type"}}); err != nil { + return err + } + } + + return e.EncodeToken(xml.EndElement{Name: start.Name}) } // JSONOutputOptions - json output specific options type JSONOutputOptions struct { - RecordDelimiter string + RecordDelimiter string + recordDelimiterSet bool +} + +// SetRecordDelimiter sets the record delimiter in the JSON output options +func (j *JSONOutputOptions) SetRecordDelimiter(val string) { + j.RecordDelimiter = val + j.recordDelimiterSet = true +} + +// MarshalXML - produces the xml representation of the JSONOutputOptions struct +func (j JSONOutputOptions) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if err := e.EncodeToken(start); err != nil { + return err + } + + if j.RecordDelimiter != "" || j.recordDelimiterSet { + if err := e.EncodeElement(j.RecordDelimiter, xml.StartElement{Name: xml.Name{Local: "RecordDelimiter"}}); err != nil { + return err + } + } + + return e.EncodeToken(xml.EndElement{Name: start.Name}) } // SelectObjectInputSerialization - input serialization parameters diff --git a/appveyor.yml b/appveyor.yml index 39a33d876..27f643c15 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,7 +23,7 @@ install: build_script: - go vet ./... - gofmt -s -l . - - golint -set_exit_status github.com/minio/minio-go/... + - golint -set_exit_status ./... - staticcheck - go test -short -v ./... - go test -short -race -v ./... From 69fefd9c12ba758d3d8a8432146ca18b177a4236 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Wed, 22 Apr 2020 01:41:00 +0000 Subject: [PATCH 142/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 9ec0a6363..ed5d682a7 100644 --- a/api.go +++ b/api.go @@ -104,7 +104,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.54" + libraryVersion = "v6.0.55" ) // User Agent should always following the below style. From 4fc8187efa80e179234fa8725128022587be9fc4 Mon Sep 17 00:00:00 2001 From: Bala FA Date: Fri, 24 Apr 2020 20:29:39 +0000 Subject: [PATCH 143/215] add tags package (#1275) --- pkg/tags/tags.go | 324 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 324 insertions(+) create mode 100644 pkg/tags/tags.go diff --git a/pkg/tags/tags.go b/pkg/tags/tags.go new file mode 100644 index 000000000..cfbaddbcd --- /dev/null +++ b/pkg/tags/tags.go @@ -0,0 +1,324 @@ +/* + * MinIO Cloud Storage, (C) 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package tags + +import ( + "encoding/xml" + "io" + "net/url" + "strings" + "unicode/utf8" +) + +// Error contains tag specific error. +type Error interface { + error + Code() string +} + +type errTag struct { + code string + message string +} + +// Code contains error code. +func (err errTag) Code() string { + return err.code +} + +// Error contains error message. +func (err errTag) Error() string { + return err.message +} + +var ( + errTooManyObjectTags = &errTag{"BadRequest", "Tags cannot be more than 10"} + errTooManyTags = &errTag{"BadRequest", "Tags cannot be more than 50"} + errInvalidTagKey = &errTag{"InvalidTag", "The TagKey you have provided is invalid"} + errInvalidTagValue = &errTag{"InvalidTag", "The TagValue you have provided is invalid"} + errDuplicateTagKey = &errTag{"InvalidTag", "Cannot provide multiple Tags with the same key"} +) + +// Tag comes with limitation as per +// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-tagging.html amd +// https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions +const ( + maxKeyLength = 128 + maxValueLength = 256 + maxObjectTagCount = 10 + maxTagCount = 50 +) + +func checkKey(key string) error { + if len(key) == 0 || utf8.RuneCountInString(key) > maxKeyLength || strings.Contains(key, "&") { + return errInvalidTagKey + } + + return nil +} + +func checkValue(value string) error { + if utf8.RuneCountInString(value) > maxValueLength || strings.Contains(value, "&") { + return errInvalidTagValue + } + + return nil +} + +// Tag denotes key and value. +type Tag struct { + Key string `xml:"Key"` + Value string `xml:"Value"` +} + +func (tag Tag) String() string { + return tag.Key + "=" + tag.Value +} + +// IsEmpty returns whether this tag is empty or not. +func (tag Tag) IsEmpty() bool { + return tag.Key == "" +} + +// Validate checks this tag. +func (tag Tag) Validate() error { + if err := checkKey(tag.Key); err != nil { + return err + } + + return checkValue(tag.Value) +} + +// MarshalXML encodes to XML data. +func (tag Tag) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if err := tag.Validate(); err != nil { + return err + } + + type subTag Tag // to avoid recursively calling MarshalXML() + return e.EncodeElement(subTag(tag), start) +} + +// UnmarshalXML decodes XML data to tag. +func (tag *Tag) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + type subTag Tag // to avoid recursively calling UnmarshalXML() + var st subTag + if err := d.DecodeElement(&st, &start); err != nil { + return err + } + + if err := Tag(st).Validate(); err != nil { + return err + } + + *tag = Tag(st) + return nil +} + +// tagSet represents list of unique tags. +type tagSet struct { + tagMap map[string]string + isObject bool +} + +func (tags tagSet) String() string { + s := []string{} + for key, value := range tags.tagMap { + s = append(s, key+"="+value) + } + + return strings.Join(s, "&") +} + +func (tags *tagSet) remove(key string) { + delete(tags.tagMap, key) +} + +func (tags *tagSet) set(key, value string, failOnExist bool) error { + if failOnExist { + if _, found := tags.tagMap[key]; found { + return errDuplicateTagKey + } + } + + if err := checkKey(key); err != nil { + return err + } + + if err := checkValue(value); err != nil { + return err + } + + if tags.isObject { + if len(tags.tagMap) == maxObjectTagCount { + return errTooManyObjectTags + } + } else if len(tags.tagMap) == maxTagCount { + return errTooManyTags + } + + tags.tagMap[key] = value + return nil +} + +func (tags tagSet) toMap() map[string]string { + m := make(map[string]string) + for key, value := range tags.tagMap { + m[key] = value + } + return m +} + +// MarshalXML encodes to XML data. +func (tags tagSet) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + tagList := struct { + Tags []Tag `xml:"Tag"` + }{} + + for key, value := range tags.tagMap { + tagList.Tags = append(tagList.Tags, Tag{key, value}) + } + + return e.EncodeElement(tagList, start) +} + +// UnmarshalXML decodes XML data to tag list. +func (tags *tagSet) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + tagList := struct { + Tags []Tag `xml:"Tag"` + }{} + + if err := d.DecodeElement(&tagList, &start); err != nil { + return err + } + + if tags.isObject { + if len(tagList.Tags) > maxObjectTagCount { + return errTooManyObjectTags + } + } else if len(tagList.Tags) > maxTagCount { + return errTooManyTags + } + + m := map[string]string{} + for _, tag := range tagList.Tags { + if _, found := m[tag.Key]; found { + return errDuplicateTagKey + } + + m[tag.Key] = tag.Value + } + + tags.tagMap = m + return nil +} + +type tagging struct { + XMLName xml.Name `xml:"Tagging"` + TagSet *tagSet `xml:"TagSet"` +} + +// Tags is list of tags of XML request/response as per +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketTagging.html#API_GetBucketTagging_RequestBody +type Tags tagging + +func (tags Tags) String() string { + return tags.TagSet.String() +} + +// Remove removes a tag by its key. +func (tags *Tags) Remove(key string) { + tags.TagSet.remove(key) +} + +// Set sets new tag. +func (tags *Tags) Set(key, value string) error { + return tags.TagSet.set(key, value, false) +} + +// ToMap returns copy of tags. +func (tags Tags) ToMap() map[string]string { + return tags.TagSet.toMap() +} + +// NewTags creates Tags from tagMap, If isObject is set, it validates for object tags. +func NewTags(tagMap map[string]string, isObject bool) (*Tags, error) { + tagging := &Tags{ + TagSet: &tagSet{ + tagMap: make(map[string]string), + isObject: isObject, + }, + } + + for key, value := range tagMap { + if err := tagging.TagSet.set(key, value, true); err != nil { + return nil, err + } + } + + return tagging, nil +} + +func unmarshalXML(reader io.Reader, isObject bool) (*Tags, error) { + tagging := &Tags{ + TagSet: &tagSet{ + tagMap: make(map[string]string), + isObject: isObject, + }, + } + + if err := xml.NewDecoder(reader).Decode(tagging); err != nil { + return nil, err + } + + return tagging, nil +} + +// ParseBucketXML decodes XML data of tags in reader specified in +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketTagging.html#API_PutBucketTagging_RequestSyntax. +func ParseBucketXML(reader io.Reader) (*Tags, error) { + return unmarshalXML(reader, false) +} + +// ParseObjectXML decodes XML data of tags in reader specified in +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectTagging.html#API_PutObjectTagging_RequestSyntax +func ParseObjectXML(reader io.Reader) (*Tags, error) { + return unmarshalXML(reader, true) +} + +// ParseObjectTags decodes HTTP query formatted string into tags. A query formatted string is like "key1=value1&key2=value2". +func ParseObjectTags(s string) (*Tags, error) { + values, err := url.ParseQuery(s) + if err != nil { + return nil, err + } + + tagging := &Tags{ + TagSet: &tagSet{ + tagMap: make(map[string]string), + isObject: true, + }, + } + + for key := range values { + if err := tagging.TagSet.set(key, values.Get(key), true); err != nil { + return nil, err + } + } + + return tagging, nil +} From 7506d2996b2298495f087e7f033f5c54405839d2 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Sat, 25 Apr 2020 02:11:15 +0530 Subject: [PATCH 144/215] fix: Use jsoniter instead of encoding/json (#1276) --- api-notification.go | 3 ++- functional_tests.go | 4 ++-- go.mod | 1 + go.sum | 13 +++++++++++++ pkg/credentials/file_minio_client.go | 4 +++- pkg/credentials/iam_aws.go | 7 ++++--- pkg/encrypt/server-side.go | 3 ++- pkg/policy/bucket-policy-condition_test.go | 4 +++- pkg/policy/bucket-policy.go | 5 +++-- pkg/policy/bucket-policy_test.go | 1 - pkg/set/stringset.go | 5 ++++- 11 files changed, 37 insertions(+), 13 deletions(-) diff --git a/api-notification.go b/api-notification.go index 57ac1ed37..8514461d7 100644 --- a/api-notification.go +++ b/api-notification.go @@ -20,11 +20,11 @@ package minio import ( "bufio" "context" - "encoding/json" "net/http" "net/url" "time" + jsoniter "github.com/json-iterator/go" "github.com/minio/minio-go/v6/pkg/s3utils" ) @@ -203,6 +203,7 @@ func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, even // Use a higher buffer to support unexpected // caching done by proxies bio.Buffer(notificationEventBuffer, notificationCapacity) + var json = jsoniter.ConfigCompatibleWithStandardLibrary // Unmarshal each line, returns marshalled values. for bio.Scan() { diff --git a/functional_tests.go b/functional_tests.go index ca66bc6fd..bd7683c18 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -22,7 +22,6 @@ package main import ( "bytes" "context" - "encoding/json" "errors" "fmt" "io" @@ -40,6 +39,7 @@ import ( "time" "github.com/dustin/go-humanize" + jsoniter "github.com/json-iterator/go" log "github.com/sirupsen/logrus" "github.com/minio/minio-go/v6" @@ -75,7 +75,7 @@ func (f *mintJSONFormatter) Format(entry *log.Entry) ([]byte, error) { data[k] = v } } - + var json = jsoniter.ConfigCompatibleWithStandardLibrary serialized, err := json.Marshal(data) if err != nil { return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err) diff --git a/go.mod b/go.mod index fe6d38bc6..fe43a735e 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.12 require ( github.com/dustin/go-humanize v1.0.0 // indirect + github.com/json-iterator/go v1.1.9 github.com/minio/sha256-simd v0.1.1 github.com/mitchellh/go-homedir v1.1.0 github.com/sirupsen/logrus v1.5.0 // indirect diff --git a/go.sum b/go.sum index 57de3a2b9..7461ddbf1 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,13 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -10,6 +15,11 @@ github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKU github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q= github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= @@ -17,7 +27,10 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykE github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/pkg/credentials/file_minio_client.go b/pkg/credentials/file_minio_client.go index 8adb5dcb1..ca6db005b 100644 --- a/pkg/credentials/file_minio_client.go +++ b/pkg/credentials/file_minio_client.go @@ -18,12 +18,12 @@ package credentials import ( - "encoding/json" "io/ioutil" "os" "path/filepath" "runtime" + jsoniter "github.com/json-iterator/go" homedir "github.com/mitchellh/go-homedir" ) @@ -122,6 +122,8 @@ type config struct { // returned if it fails to read from the file. func loadAlias(filename, alias string) (hostConfig, error) { cfg := &config{} + var json = jsoniter.ConfigCompatibleWithStandardLibrary + configBytes, err := ioutil.ReadFile(filename) if err != nil { return hostConfig{}, err diff --git a/pkg/credentials/iam_aws.go b/pkg/credentials/iam_aws.go index 60f78f808..a7eb8cb9a 100644 --- a/pkg/credentials/iam_aws.go +++ b/pkg/credentials/iam_aws.go @@ -19,7 +19,6 @@ package credentials import ( "bufio" - "encoding/json" "errors" "fmt" "io/ioutil" @@ -29,6 +28,8 @@ import ( "os" "path" "time" + + jsoniter "github.com/json-iterator/go" ) // DefaultExpiryWindow - Default expiry window. @@ -227,7 +228,7 @@ func getEcsTaskCredentials(client *http.Client, endpoint string) (ec2RoleCredRes } respCreds := ec2RoleCredRespBody{} - if err := json.NewDecoder(resp.Body).Decode(&respCreds); err != nil { + if err := jsoniter.NewDecoder(resp.Body).Decode(&respCreds); err != nil { return ec2RoleCredRespBody{}, err } @@ -283,7 +284,7 @@ func getCredentials(client *http.Client, endpoint string) (ec2RoleCredRespBody, } respCreds := ec2RoleCredRespBody{} - if err := json.NewDecoder(resp.Body).Decode(&respCreds); err != nil { + if err := jsoniter.NewDecoder(resp.Body).Decode(&respCreds); err != nil { return ec2RoleCredRespBody{}, err } diff --git a/pkg/encrypt/server-side.go b/pkg/encrypt/server-side.go index cabc65067..5276f63fc 100644 --- a/pkg/encrypt/server-side.go +++ b/pkg/encrypt/server-side.go @@ -20,10 +20,10 @@ package encrypt import ( "crypto/md5" "encoding/base64" - "encoding/json" "errors" "net/http" + jsoniter "github.com/json-iterator/go" "golang.org/x/crypto/argon2" ) @@ -101,6 +101,7 @@ func NewSSEKMS(keyID string, context interface{}) (ServerSide, error) { if context == nil { return kms{key: keyID, hasContext: false}, nil } + var json = jsoniter.ConfigCompatibleWithStandardLibrary serializedContext, err := json.Marshal(context) if err != nil { return nil, err diff --git a/pkg/policy/bucket-policy-condition_test.go b/pkg/policy/bucket-policy-condition_test.go index 2fc9baa60..db24cf364 100644 --- a/pkg/policy/bucket-policy-condition_test.go +++ b/pkg/policy/bucket-policy-condition_test.go @@ -18,12 +18,14 @@ package policy import ( - "encoding/json" "testing" + jsoniter "github.com/json-iterator/go" "github.com/minio/minio-go/v6/pkg/set" ) +var json = jsoniter.ConfigCompatibleWithStandardLibrary + // ConditionKeyMap.Add() is called and the result is validated. func TestConditionKeyMapAdd(t *testing.T) { condKeyMap := make(ConditionKeyMap) diff --git a/pkg/policy/bucket-policy.go b/pkg/policy/bucket-policy.go index f2c728924..f318be2a0 100644 --- a/pkg/policy/bucket-policy.go +++ b/pkg/policy/bucket-policy.go @@ -18,11 +18,11 @@ package policy import ( - "encoding/json" "errors" "reflect" "strings" + jsoniter "github.com/json-iterator/go" "github.com/minio/minio-go/v6/pkg/set" ) @@ -88,7 +88,8 @@ type User struct { // the reason is that Principal can take a json struct represented by // User string but it can also take a string. func (u *User) UnmarshalJSON(data []byte) error { - // Try to unmarshal data in a struct equal to User, we need it + // Try to unmarshal data in a struct equal to User, + var json = jsoniter.ConfigCompatibleWithStandardLibrary // to avoid infinite recursive call of this function type AliasUser User var au AliasUser diff --git a/pkg/policy/bucket-policy_test.go b/pkg/policy/bucket-policy_test.go index a514eaa0f..7d0a9cbfb 100644 --- a/pkg/policy/bucket-policy_test.go +++ b/pkg/policy/bucket-policy_test.go @@ -18,7 +18,6 @@ package policy import ( - "encoding/json" "fmt" "reflect" "testing" diff --git a/pkg/set/stringset.go b/pkg/set/stringset.go index e220271bb..c35e58e1a 100644 --- a/pkg/set/stringset.go +++ b/pkg/set/stringset.go @@ -18,14 +18,17 @@ package set import ( - "encoding/json" "fmt" "sort" + + jsoniter "github.com/json-iterator/go" ) // StringSet - uses map as set of strings. type StringSet map[string]struct{} +var json = jsoniter.ConfigCompatibleWithStandardLibrary + // ToSlice - returns StringSet as string slice. func (set StringSet) ToSlice() []string { keys := make([]string, 0, len(set)) From f303b4f86edf02301e897e0bda807633b1ef7b61 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Sat, 25 Apr 2020 00:49:52 +0000 Subject: [PATCH 145/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index ed5d682a7..f9bdbb9cd 100644 --- a/api.go +++ b/api.go @@ -104,7 +104,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.55" + libraryVersion = "v6.0.56" ) // User Agent should always following the below style. From 89eebdef2af042e454af57588e4837c63f8a8285 Mon Sep 17 00:00:00 2001 From: Bala FA Date: Sat, 25 Apr 2020 08:14:27 +0000 Subject: [PATCH 146/215] add Parse in tags pkg (#1279) --- pkg/tags/tags.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/tags/tags.go b/pkg/tags/tags.go index cfbaddbcd..6ef5628e9 100644 --- a/pkg/tags/tags.go +++ b/pkg/tags/tags.go @@ -300,8 +300,9 @@ func ParseObjectXML(reader io.Reader) (*Tags, error) { return unmarshalXML(reader, true) } -// ParseObjectTags decodes HTTP query formatted string into tags. A query formatted string is like "key1=value1&key2=value2". -func ParseObjectTags(s string) (*Tags, error) { +// Parse decodes HTTP query formatted string into tags which is limited by isObject. +// A query formatted string is like "key1=value1&key2=value2". +func Parse(s string, isObject bool) (*Tags, error) { values, err := url.ParseQuery(s) if err != nil { return nil, err @@ -310,7 +311,7 @@ func ParseObjectTags(s string) (*Tags, error) { tagging := &Tags{ TagSet: &tagSet{ tagMap: make(map[string]string), - isObject: true, + isObject: isObject, }, } @@ -322,3 +323,8 @@ func ParseObjectTags(s string) (*Tags, error) { return tagging, nil } + +// ParseObjectTags decodes HTTP query formatted string into tags. A query formatted string is like "key1=value1&key2=value2". +func ParseObjectTags(s string) (*Tags, error) { + return Parse(s, true) +} From ae35e52f26c14368f967da2c0f4b4c165d5a3808 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 27 Apr 2020 21:02:25 -0700 Subject: [PATCH 147/215] add github workflows, deprecate travis and appveyor --- .github/workflows/go-windows.yml | 45 ++++++++ .github/workflows/go.yml | 47 +++++++++ .gitignore | 1 + .golangci.yml | 16 +++ .travis.yml | 30 ------ Makefile | 6 +- api-datatypes.go | 2 +- api-notification.go | 2 +- api-object-lock.go | 2 +- appveyor.yml | 35 ------- bucket-cache.go | 2 +- core_test.go | 2 +- functional_tests.go | 175 ++++++++++++++++++++----------- pkg/credentials/iam_aws_test.go | 3 - post-policy.go | 4 +- transport.go | 30 +++++- 16 files changed, 261 insertions(+), 141 deletions(-) create mode 100644 .github/workflows/go-windows.yml create mode 100644 .github/workflows/go.yml create mode 100644 .golangci.yml delete mode 100644 .travis.yml delete mode 100644 appveyor.yml diff --git a/.github/workflows/go-windows.yml b/.github/workflows/go-windows.yml new file mode 100644 index 000000000..b0824f320 --- /dev/null +++ b/.github/workflows/go-windows.yml @@ -0,0 +1,45 @@ +name: Go + +on: + pull_request: + branches: + - master + push: + branches: + - master + +jobs: + build: + name: Test on Go ${{ matrix.go-version }} and ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + go-version: [1.13.x, 1.14.x] + os: [windows-latest] + steps: + - name: Set up Go ${{ matrix.go-version }} on ${{ matrix.os }} + uses: actions/setup-go@v1 + with: + go-version: ${{ matrix.go-version }} + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v1 + + - name: Build on ${{ matrix.os }} + env: + MINT_MODE: core + SERVER_ENDPOINT: localhost:9000 + ACCESS_KEY: minio + SECRET_KEY: minio123 + ENABLE_HTTPS: 1 + MINIO_ACCESS_KEY: minio + MINIO_SECRET_KEY: minio123 + GO111MODULE: on + run: | + New-Item -ItemType Directory -Path "$env:temp/certs-dir" + Copy-Item -Path testcerts\* -Destination "$env:temp/certs-dir" + Invoke-WebRequest -Uri https://dl.minio.io/server/minio/release/windows-amd64/minio.exe -OutFile $HOME/minio.exe + Start-Process -NoNewWindow -FilePath "$HOME/minio.exe" -ArgumentList "-S", "$env:temp/certs-dir", "server", "$env:temp/fs" + $env:SSL_CERT_FILE = "$env:temp/certs-dir/public.crt" + go run functional_tests.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 000000000..efd5ec943 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,47 @@ +name: Go + +on: + pull_request: + branches: + - master + push: + branches: + - master + +jobs: + build: + name: Test on Go ${{ matrix.go-version }} and ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + go-version: [1.13.x, 1.14.x] + os: [ubuntu-latest] + steps: + - name: Set up Go ${{ matrix.go-version }} on ${{ matrix.os }} + uses: actions/setup-go@v1 + with: + go-version: ${{ matrix.go-version }} + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v1 + + - name: Build on ${{ matrix.os }} + env: + MINT_MODE: full + SERVER_ENDPOINT: localhost:9000 + ACCESS_KEY: minio + SECRET_KEY: minio123 + ENABLE_HTTPS: 1 + MINIO_ACCESS_KEY: minio + MINIO_SECRET_KEY: minio123 + GO111MODULE: on + SSL_CERT_FILE: /tmp/certs-dir/public.crt + run: | + sudo apt-get install devscripts + wget -O /tmp/minio https://dl.min.io/server/minio/release/linux-amd64/minio + chmod +x /tmp/minio + mkdir -p /tmp/certs-dir + cp testcerts/* /tmp/certs-dir + /tmp/minio server --quiet -S /tmp/certs-dir /tmp/fs & + make diff --git a/.gitignore b/.gitignore index fa967abd7..8081bd0ff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *~ *.test validator +golangci-lint \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 000000000..7d1dd3352 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,16 @@ +linters-settings: + misspell: + locale: US + +linters: + disable-all: true + enable: + - typecheck + - goimports + - misspell + - govet + - golint + - ineffassign + - gosimple + - deadcode + - structcheck diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index e50cefee7..000000000 --- a/.travis.yml +++ /dev/null @@ -1,30 +0,0 @@ -sudo: false -language: go - -os: -- linux - -env: -- ARCH=x86_64 - -go: -- 1.13.x -- tip - -matrix: - fast_finish: true - allow_failures: - - go: tip - -before_install: - - sudo apt-get install devscripts - - mkdir /tmp/minio - - (cd /tmp/minio; GO111MODULE=on go get github.com/minio/minio) - - sudo cp testcerts/public.crt /usr/local/share/ca-certificates/ - - sudo update-ca-certificates - - MINIO_ACCESS_KEY=minio MINIO_SECRET_KEY=minio123 ${GOPATH}/bin/minio server --compat --quiet --certs-dir testcerts data 2>&1 > minio.log & - -script: - - diff -au <(gofmt -d .) <(printf "") - - diff -au <(licensecheck --check '.go$' --recursive --lines 0 * | grep -v -w 'Apache (v2.0)') <(printf "") - - make diff --git a/Makefile b/Makefile index 18a74665c..afe8d978f 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,11 @@ all: checks .PHONY: examples docs -checks: vet test examples functional-test +checks: diff vet test examples functional-test + +diff: + @curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b . v1.21.0 + @./golangci-lint run --timeout=5m --config ./.golangci.yml vet: @GO111MODULE=on go vet ./... diff --git a/api-datatypes.go b/api-datatypes.go index 58e3c5e0e..9db73736a 100644 --- a/api-datatypes.go +++ b/api-datatypes.go @@ -41,7 +41,7 @@ type StringMap map[string]string // // The fact this function is on the pointer of Map is important, so that // if m is nil it can be initialized, which is often the case if m is -// nested in another xml structurel. This is also why the first thing done +// nested in another xml structural. This is also why the first thing done // on the first line is initialize it. func (m *StringMap) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { *m = StringMap{} diff --git a/api-notification.go b/api-notification.go index 8514461d7..cae3b14e4 100644 --- a/api-notification.go +++ b/api-notification.go @@ -205,7 +205,7 @@ func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, even bio.Buffer(notificationEventBuffer, notificationCapacity) var json = jsoniter.ConfigCompatibleWithStandardLibrary - // Unmarshal each line, returns marshalled values. + // Unmarshal each line, returns marshaled values. for bio.Scan() { var notificationInfo NotificationInfo if err = json.Unmarshal(bio.Bytes(), ¬ificationInfo); err != nil { diff --git a/api-object-lock.go b/api-object-lock.go index 14168ef39..bdd2b07c4 100644 --- a/api-object-lock.go +++ b/api-object-lock.go @@ -33,7 +33,7 @@ import ( type RetentionMode string const ( - // Governance - goverance mode. + // Governance - governance mode. Governance RetentionMode = "GOVERNANCE" // Compliance - compliance mode. diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 27f643c15..000000000 --- a/appveyor.yml +++ /dev/null @@ -1,35 +0,0 @@ -# version format -version: "{build}" - -# Operating system (build VM template) -os: Windows Server 2012 R2 - -clone_folder: c:\gopath\src\github.com\minio\minio-go - -# environment variables -environment: - GOPATH: c:\gopath - GO111MODULE: on - -# scripts that run after cloning repository -install: - - set PATH=%GOPATH%\bin;c:\go\bin;%PATH% - - go version - - go env - - go get golang.org/x/lint/golint - - go get honnef.co/go/tools/cmd/staticcheck - -# to run your custom scripts instead of automatic MSBuild -build_script: - - go vet ./... - - gofmt -s -l . - - golint -set_exit_status ./... - - staticcheck - - go test -short -v ./... - - go test -short -race -v ./... - -# to disable automatic tests -test: off - -# to disable deployment -deploy: off diff --git a/bucket-cache.go b/bucket-cache.go index 6727840ae..305367bb5 100644 --- a/bucket-cache.go +++ b/bucket-cache.go @@ -187,7 +187,7 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro var urlStr string - //only suport Aliyun OSS for virtual hosted path, compatible Amazon & Google Endpoint + //only support Aliyun OSS for virtual hosted path, compatible Amazon & Google Endpoint if isVirtualHost && s3utils.IsAliyunOSSEndpoint(targetURL) { urlStr = c.endpointURL.Scheme + "://" + bucketName + "." + targetURL.Host + "/?location" } else { diff --git a/core_test.go b/core_test.go index 467c949f3..e335dd43e 100644 --- a/core_test.go +++ b/core_test.go @@ -241,7 +241,7 @@ func TestGetObjectCore(t *testing.T) { t.Fatal("Error: ", err) } if contentLength != contentLengthValue { - t.Fatalf("Error: Content Length in response header %v, not equal to set content lenght %v\n", contentLengthValue, contentLength) + t.Fatalf("Error: Content Length in response header %v, not equal to set content length %v\n", contentLengthValue, contentLength) } err = c.RemoveObject(bucketName, objectName) diff --git a/functional_tests.go b/functional_tests.go index bd7683c18..70b757156 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -1555,11 +1555,7 @@ func testFPutObject() { return } - if err = os.Remove(fName + ".gtar"); err != nil { - logError(testName, function, args, startTime, "", "File remove failed", err) - return - } - + os.Remove(fName + ".gtar") successLogger(testName, function, args, startTime).Info() } @@ -2513,8 +2509,30 @@ func testPresignedPostPolicy() { } writer.Close() + transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS))) + if err != nil { + logError(testName, function, args, startTime, "", "DefaultTransport failed", err) + return + } + + httpClient := &http.Client{ + // Setting a sensible time out of 30secs to wait for response + // headers. Request is pro-actively canceled after 30secs + // with no response. + Timeout: 30 * time.Second, + Transport: transport, + } + + req, err := http.NewRequest(http.MethodPost, presignedPostPolicyURL.String(), bytes.NewReader(formBuf.Bytes())) + if err != nil { + logError(testName, function, args, startTime, "", "Http request failed", err) + return + } + + req.Header.Set("Content-Type", writer.FormDataContentType()) + // make post request with correct form data - res, err := http.Post(presignedPostPolicyURL.String(), writer.FormDataContentType(), bytes.NewReader(formBuf.Bytes())) + res, err := httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "Http request failed", err) return @@ -3768,10 +3786,7 @@ func testSSECEncryptionFPut() { return } - if err = os.Remove(fileName); err != nil { - logError(testName, function, args, startTime, "", "File remove failed", err) - return - } + os.Remove(fileName) } // Delete all objects and buckets @@ -4009,10 +4024,7 @@ func testSSES3EncryptionFPut() { return } - if err = os.Remove(fileName); err != nil { - logError(testName, function, args, startTime, "", "File remove failed", err) - return - } + os.Remove(fileName) } // Delete all objects and buckets @@ -4541,8 +4553,29 @@ func testFunctional() { logError(testName, function, args, startTime, "", "PresignedHeadObject failed", err) return } + + transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS))) + if err != nil { + logError(testName, function, args, startTime, "", "DefaultTransport failed", err) + return + } + + httpClient := &http.Client{ + // Setting a sensible time out of 30secs to wait for response + // headers. Request is pro-actively canceled after 30secs + // with no response. + Timeout: 30 * time.Second, + Transport: transport, + } + + req, err := http.NewRequest(http.MethodHead, presignedHeadURL.String(), nil) + if err != nil { + logError(testName, function, args, startTime, "", "PresignedHeadObject request was incorrect", err) + return + } + // Verify if presigned url works. - resp, err := http.Head(presignedHeadURL.String()) + resp, err := httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect", err) return @@ -4586,7 +4619,13 @@ func testFunctional() { } // Verify if presigned url works. - resp, err = http.Get(presignedGetURL.String()) + req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil) + if err != nil { + logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err) + return + } + + resp, err = httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) return @@ -4621,8 +4660,15 @@ func testFunctional() { logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) return } + // Verify if presigned url works. - resp, err = http.Get(presignedGetURL.String()) + req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil) + if err != nil { + logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err) + return + } + + resp, err = httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) return @@ -4674,18 +4720,12 @@ func testFunctional() { buf = bytes.Repeat([]byte("g"), 1<<19) - req, err := http.NewRequest("PUT", presignedPutURL.String(), bytes.NewReader(buf)) + req, err = http.NewRequest(http.MethodPut, presignedPutURL.String(), bytes.NewReader(buf)) if err != nil { logError(testName, function, args, startTime, "", "Couldn't make HTTP request with PresignedPutObject URL", err) return } - httpClient := &http.Client{ - // Setting a sensible time out of 30secs to wait for response - // headers. Request is pro-actively cancelled after 30secs - // with no response. - Timeout: 30 * time.Second, - Transport: http.DefaultTransport, - } + resp, err = httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "PresignedPutObject failed", err) @@ -4766,14 +4806,8 @@ func testFunctional() { return } - if err = os.Remove(fileName); err != nil { - logError(testName, function, args, startTime, "", "File Remove failed", err) - return - } - if err = os.Remove(fileName + "-f"); err != nil { - logError(testName, function, args, startTime, "", "File Remove failed", err) - return - } + os.Remove(fileName) + os.Remove(fileName + "-f") successLogger(testName, functionAll, args, startTime).Info() } @@ -5267,7 +5301,7 @@ func testFPutObjectV2() { // Add extension to temp file name fileName := file.Name() - err = os.Rename(file.Name(), fileName+".gtar") + err = os.Rename(fileName, fileName+".gtar") if err != nil { logError(testName, function, args, startTime, "", "Rename failed", err) return @@ -5335,11 +5369,7 @@ func testFPutObjectV2() { return } - err = os.Remove(fileName + ".gtar") - if err != nil { - logError(testName, function, args, startTime, "", "File remove failed", err) - return - } + os.Remove(fileName + ".gtar") successLogger(testName, function, args, startTime).Info() } @@ -9211,8 +9241,29 @@ func testFunctionalV2() { logError(testName, function, args, startTime, "", "PresignedHeadObject failed", err) return } + + transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS))) + if err != nil { + logError(testName, function, args, startTime, "", "DefaultTransport failed", err) + return + } + + httpClient := &http.Client{ + // Setting a sensible time out of 30secs to wait for response + // headers. Request is pro-actively canceled after 30secs + // with no response. + Timeout: 30 * time.Second, + Transport: transport, + } + + req, err := http.NewRequest(http.MethodHead, presignedHeadURL.String(), nil) + if err != nil { + logError(testName, function, args, startTime, "", "PresignedHeadObject URL head request failed", err) + return + } + // Verify if presigned url works. - resp, err := http.Head(presignedHeadURL.String()) + resp, err := httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "PresignedHeadObject URL head request failed", err) return @@ -9240,12 +9291,20 @@ func testFunctionalV2() { logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) return } + // Verify if presigned url works. - resp, err = http.Get(presignedGetURL.String()) + req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil) if err != nil { - logError(testName, function, args, startTime, "", "PresignedGetObject URL GET request failed", err) + logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err) + return + } + + resp, err = httpClient.Do(req) + if err != nil { + logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) return } + if resp.StatusCode != http.StatusOK { logError(testName, function, args, startTime, "", "PresignedGetObject URL returns status "+string(resp.StatusCode), err) return @@ -9271,12 +9330,20 @@ func testFunctionalV2() { logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) return } + // Verify if presigned url works. - resp, err = http.Get(presignedGetURL.String()) + req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil) + if err != nil { + logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err) + return + } + + resp, err = httpClient.Do(req) if err != nil { - logError(testName, function, args, startTime, "", "PresignedGetObject URL GET request failed", err) + logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) return } + if resp.StatusCode != http.StatusOK { logError(testName, function, args, startTime, "", "PresignedGetObject URL returns status "+string(resp.StatusCode), err) return @@ -9312,18 +9379,12 @@ func testFunctionalV2() { // Generate data more than 32K buf = bytes.Repeat([]byte("1"), rand.Intn(1<<10)+32*1024) - req, err := http.NewRequest("PUT", presignedPutURL.String(), bytes.NewReader(buf)) + req, err = http.NewRequest(http.MethodPut, presignedPutURL.String(), bytes.NewReader(buf)) if err != nil { logError(testName, function, args, startTime, "", "HTTP request to PresignedPutObject URL failed", err) return } - httpClient := &http.Client{ - // Setting a sensible time out of 30secs to wait for response - // headers. Request is pro-actively cancelled after 30secs - // with no response. - Timeout: 30 * time.Second, - Transport: http.DefaultTransport, - } + resp, err = httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "HTTP request to PresignedPutObject URL failed", err) @@ -9360,14 +9421,8 @@ func testFunctionalV2() { return } - if err = os.Remove(fileName); err != nil { - logError(testName, function, args, startTime, "", "File remove failed", err) - return - } - if err = os.Remove(fileName + "-f"); err != nil { - logError(testName, function, args, startTime, "", "File removes failed", err) - return - } + os.Remove(fileName) + os.Remove(fileName + "-f") successLogger(testName, functionAll, args, startTime).Info() } diff --git a/pkg/credentials/iam_aws_test.go b/pkg/credentials/iam_aws_test.go index 201514629..a7c3d3368 100644 --- a/pkg/credentials/iam_aws_test.go +++ b/pkg/credentials/iam_aws_test.go @@ -135,9 +135,6 @@ func TestIAMMalformedEndpoint(t *testing.T) { if err == nil { t.Fatal("Unexpected should fail here") } - if err.Error() != `parse %%%%: invalid URL escape "%%%"` { - t.Fatalf("Expected parse %%%%%%%%: invalid URL escape \"%%%%%%\", got %s", err) - } } func TestIAMFailServer(t *testing.T) { diff --git a/post-policy.go b/post-policy.go index 017916f06..5b84486a4 100644 --- a/post-policy.go +++ b/post-policy.go @@ -263,7 +263,7 @@ func (p PostPolicy) String() string { return string(p.marshalJSON()) } -// marshalJSON - Provides Marshalled JSON in bytes. +// marshalJSON - Provides Marshaled JSON in bytes. func (p PostPolicy) marshalJSON() []byte { expirationStr := `"expiration":"` + p.expiration.Format(expirationDateFormat) + `"` var conditionsStr string @@ -285,7 +285,7 @@ func (p PostPolicy) marshalJSON() []byte { return []byte(retStr) } -// base64 - Produces base64 of PostPolicy's Marshalled json. +// base64 - Produces base64 of PostPolicy's Marshaled json. func (p PostPolicy) base64() string { return base64.StdEncoding.EncodeToString(p.marshalJSON()) } diff --git a/transport.go b/transport.go index fb624854b..4a1e17488 100644 --- a/transport.go +++ b/transport.go @@ -21,11 +21,23 @@ package minio import ( "crypto/tls" + "crypto/x509" + "io/ioutil" "net" "net/http" + "os" "time" ) +// mustGetSystemCertPool - return system CAs or empty pool in case of error (or windows) +func mustGetSystemCertPool() *x509.CertPool { + pool, err := x509.SystemCertPool() + if err != nil { + return x509.NewCertPool() + } + return pool +} + // DefaultTransport - this default transport is similar to // http.DefaultTransport but with additional param DisableCompression // is set to true to avoid decompressing content with 'gzip' encoding. @@ -36,12 +48,12 @@ var DefaultTransport = func(secure bool) (http.RoundTripper, error) { Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, }).DialContext, - MaxIdleConns: 1024, - MaxIdleConnsPerHost: 1024, - ResponseHeaderTimeout: 60 * time.Second, - IdleConnTimeout: 60 * time.Second, + MaxIdleConns: 256, + MaxIdleConnsPerHost: 16, + ResponseHeaderTimeout: time.Minute, + IdleConnTimeout: time.Minute, TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, + ExpectContinueTimeout: 10 * time.Second, // Set this value so that the underlying transport round-tripper // doesn't try to auto decode the body of objects with // content-encoding set to `gzip`. @@ -58,6 +70,14 @@ var DefaultTransport = func(secure bool) (http.RoundTripper, error) { // Can't use TLSv1.1 because of RC4 cipher usage MinVersion: tls.VersionTLS12, } + if f := os.Getenv("SSL_CERT_FILE"); f != "" { + rootCAs := mustGetSystemCertPool() + data, err := ioutil.ReadFile(f) + if err == nil { + rootCAs.AppendCertsFromPEM(data) + } + tr.TLSClientConfig.RootCAs = rootCAs + } } return tr, nil } From c350b2efb206b76bbb35261de4312c79b7bd5aae Mon Sep 17 00:00:00 2001 From: Bala FA Date: Tue, 28 Apr 2020 17:10:17 +0000 Subject: [PATCH 148/215] add bucket tagging apis (#1273) --- api-bucket-tagging.go | 147 +++++++++++++++++++++++++++++ api-object-tagging.go | 20 +--- api-s3-datatypes.go | 16 ---- docs/API.md | 138 ++++++++++++++++++++------- examples/s3/getbuckettagging.go | 47 +++++++++ examples/s3/removebuckettagging.go | 45 +++++++++ examples/s3/setbuckettagging.go | 55 +++++++++++ 7 files changed, 404 insertions(+), 64 deletions(-) create mode 100644 api-bucket-tagging.go create mode 100644 examples/s3/getbuckettagging.go create mode 100644 examples/s3/removebuckettagging.go create mode 100644 examples/s3/setbuckettagging.go diff --git a/api-bucket-tagging.go b/api-bucket-tagging.go new file mode 100644 index 000000000..0ebc4b27f --- /dev/null +++ b/api-bucket-tagging.go @@ -0,0 +1,147 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "context" + "encoding/xml" + "errors" + "io" + "io/ioutil" + "net/http" + "net/url" + + "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/tags" +) + +// GetBucketTagging gets tagging configuration for a bucket. +func (c Client) GetBucketTagging(bucketName string) (*tags.Tags, error) { + return c.GetBucketTaggingWithContext(context.Background(), bucketName) +} + +// GetBucketTaggingWithContext gets tagging configuration for a bucket with a context to control cancellations and timeouts. +func (c Client) GetBucketTaggingWithContext(ctx context.Context, bucketName string) (*tags.Tags, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return nil, err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("tagging", "") + + // Execute GET on bucket to get tagging configuration. + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + }) + + defer closeResponse(resp) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return nil, httpRespToErrorResponse(resp, bucketName, "") + } + + defer io.Copy(ioutil.Discard, resp.Body) + return tags.ParseBucketXML(resp.Body) +} + +// SetBucketTagging sets tagging configuration for a bucket. +func (c Client) SetBucketTagging(bucketName string, tags *tags.Tags) error { + return c.SetBucketTaggingWithContext(context.Background(), bucketName, tags) +} + +// SetBucketTaggingWithContext sets tagging configuration for a bucket with a context to control cancellations and timeouts. +func (c Client) SetBucketTaggingWithContext(ctx context.Context, bucketName string, tags *tags.Tags) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + if tags == nil { + return errors.New("nil tags passed") + } + + buf, err := xml.Marshal(tags) + if err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("tagging", "") + + // Content-length is mandatory to set a default encryption configuration + reqMetadata := requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: bytes.NewReader(buf), + contentLength: int64(len(buf)), + contentMD5Base64: sumMD5Base64(buf), + } + + // Execute PUT on bucket to put tagging configuration. + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, bucketName, "") + } + return nil +} + +// DeleteBucketTagging removes tagging configuration for a bucket. +func (c Client) DeleteBucketTagging(bucketName string) error { + return c.DeleteBucketTaggingWithContext(context.Background(), bucketName) +} + +// DeleteBucketTaggingWithContext removes tagging configuration for a bucket with a context to control cancellations and timeouts. +func (c Client) DeleteBucketTaggingWithContext(ctx context.Context, bucketName string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("tagging", "") + + // Execute DELETE on bucket to remove tagging configuration. + resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return httpRespToErrorResponse(resp, bucketName, "") + } + return nil +} diff --git a/api-object-tagging.go b/api-object-tagging.go index 66568063c..65df4f3e7 100644 --- a/api-object-tagging.go +++ b/api-object-tagging.go @@ -26,6 +26,7 @@ import ( "net/url" "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/tags" ) // PutObjectTagging replaces or creates object tag(s) @@ -46,23 +47,12 @@ func (c Client) PutObjectTaggingWithContext(ctx context.Context, bucketName, obj urlValues := make(url.Values) urlValues.Set("tagging", "") - tags := make([]tag, 0) - for k, v := range objectTags { - t := tag{ - Key: k, - Value: v, - } - tags = append(tags, t) - } - - // Prepare Tagging struct - reqBody := tagging{ - TagSet: tagSet{ - Tags: tags, - }, + tags, err := tags.NewTags(objectTags, true) + if err != nil { + return err } - reqBytes, err := xml.Marshal(reqBody) + reqBytes, err := xml.Marshal(tags) if err != nil { return err } diff --git a/api-s3-datatypes.go b/api-s3-datatypes.go index deb56e39d..a6b125522 100644 --- a/api-s3-datatypes.go +++ b/api-s3-datatypes.go @@ -243,19 +243,3 @@ type deleteMultiObjectsResult struct { DeletedObjects []deletedObject `xml:"Deleted"` UnDeletedObjects []nonDeletedObject `xml:"Error"` } - -type tagging struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Tagging"` - TagSet tagSet -} - -type tagSet struct { - XMLName xml.Name `xml:"TagSet"` - Tags []tag -} - -type tag struct { - XMLName xml.Name `xml:"Tag"` - Key string `xml:"Key"` - Value string `xml:"Value"` -} diff --git a/docs/API.md b/docs/API.md index cac3b9a49..33f1c3244 100644 --- a/docs/API.md +++ b/docs/API.md @@ -50,39 +50,39 @@ func main() { } ``` -| Bucket operations | Object operations | Encrypted Object operations | Presigned operations | Bucket Policy/Notification Operations | Client custom settings | -| :--- | :--- | :--- | :--- | :--- | :--- | -| [`MakeBucket`](#MakeBucket) | [`GetObject`](#GetObject) | [`GetObject`](#GetObject) | [`PresignedGetObject`](#PresignedGetObject) | [`SetBucketPolicy`](#SetBucketPolicy) | [`SetAppInfo`](#SetAppInfo) | -| [`MakeBucketWithObjectLock`](#MakeBucketWithObjectLock) | [`PutObject`](#PutObject) | [`PutObject`](#PutObject) | [`PresignedPutObject`](#PresignedPutObject) | [`GetBucketPolicy`](#GetBucketPolicy) | [`SetCustomTransport`](#SetCustomTransport) | -| [`ListBuckets`](#ListBuckets) | [`CopyObject`](#CopyObject) | [`CopyObject`](#CopyObject) | [`PresignedPostPolicy`](#PresignedPostPolicy) | [`SetBucketNotification`](#SetBucketNotification) | [`TraceOn`](#TraceOn) | -| [`BucketExists`](#BucketExists) | [`StatObject`](#StatObject) | [`StatObject`](#StatObject) | | [`GetBucketNotification`](#GetBucketNotification) | [`TraceOff`](#TraceOff) | -| [`RemoveBucket`](#RemoveBucket) | [`RemoveObject`](#RemoveObject) | | | [`RemoveAllBucketNotification`](#RemoveAllBucketNotification) | [`SetS3TransferAccelerate`](#SetS3TransferAccelerate) | -| [`ListObjects`](#ListObjects) | [`RemoveObjects`](#RemoveObjects) | | | [`ListenBucketNotification`](#ListenBucketNotification) | | -| [`ListObjectsV2`](#ListObjectsV2) | [`RemoveIncompleteUpload`](#RemoveIncompleteUpload) | | | [`SetBucketLifecycle`](#SetBucketLifecycle) | | -| [`ListIncompleteUploads`](#ListIncompleteUploads) | [`FPutObject`](#FPutObject) | [`FPutObject`](#FPutObject) | | [`GetBucketLifecycle`](#GetBucketLifecycle) | | -| | [`FGetObject`](#FGetObject) | [`FGetObject`](#FGetObject) | | [`SetObjectLockConfig`](#SetObjectLockConfig) | | -| | [`ComposeObject`](#ComposeObject) | [`ComposeObject`](#ComposeObject) | | [`GetObjectLockConfig`](#GetObjectLockConfig) | | -| | [`NewSourceInfo`](#NewSourceInfo) | [`NewSourceInfo`](#NewSourceInfo) | | [`EnableVersioning`](#EnableVersioning) | | -| | [`NewDestinationInfo`](#NewDestinationInfo) | [`NewDestinationInfo`](#NewDestinationInfo) | | [`DisableVersioning`](#DisableVersioning) | | -| | [`PutObjectWithContext`](#PutObjectWithContext) | [`PutObjectWithContext`](#PutObjectWithContext) | | [`GetBucketVersioning`](#GetBucketVersioning) | -| | [`GetObjectWithContext`](#GetObjectWithContext) | [`GetObjectWithContext`](#GetObjectWithContext) | | [`SetBucketEncryption`](#SetBucketEncryption) | | -| | [`FPutObjectWithContext`](#FPutObjectWithContext) | [`FPutObjectWithContext`](#FPutObjectWithContext) | | [`GetBucketEncryption`](#GetBucketEncryption) | -| | [`FGetObjectWithContext`](#FGetObjectWithContext) | [`FGetObjectWithContext`](#FGetObjectWithContext) | | [`DeleteBucketEncryption`](#DeleteBucketEncryption) | -| | [`RemoveObjectsWithContext`](#RemoveObjectsWithContext) | | | | -| | [`RemoveObjectsWithOptions`](#RemoveObjectsWithOptions) | | | | -| | [`RemoveObjectsWithOptionsContext`](#RemoveObjectsWithOptionsContext) | | | | -| | [`RemoveObjectWithOptions`](#RemoveObjectWithOptions) | | | | -| | [`PutObjectRetention`](#PutObjectRetention) | | | | -| | [`GetObjectRetention`](#GetObjectRetention) | | | | -| | [`PutObjectLegalHold`](#PutObjectLegalHold) | | | | -| | [`GetObjectLegalHold`](#GetObjectLegalHold) | | | | -| | [`SelectObjectContent`](#SelectObjectContent) | | -| | [`PutObjectTagging`](#PutObjectTagging) | | -| | [`PutObjectTaggingWithContext`](#PutObjectTaggingWithContext) | | -| | [`GetObjectTagging`](#GetObjectTagging) | | -| | [`GetObjectTaggingWithContext`](#GetObjectTaggingWithContext) | | -| | [`RemoveObjectTagging`](#RemoveObjectTagging) | | -| | [`RemoveObjectTaggingWithContext`](#RemoveObjectTaggingWithContext) | | +| Bucket operations | Object operations | Encrypted Object operations | Presigned operations | Bucket Policy/Notification Operations | Client custom settings | +| :--- | :--- | :--- | :--- | :--- | :--- | +| [`MakeBucket`](#MakeBucket) | [`GetObject`](#GetObject) | [`GetObject`](#GetObject) | [`PresignedGetObject`](#PresignedGetObject) | [`SetBucketPolicy`](#SetBucketPolicy) | [`SetAppInfo`](#SetAppInfo) | +| [`MakeBucketWithObjectLock`](#MakeBucketWithObjectLock) | [`PutObject`](#PutObject) | [`PutObject`](#PutObject) | [`PresignedPutObject`](#PresignedPutObject) | [`GetBucketPolicy`](#GetBucketPolicy) | [`SetCustomTransport`](#SetCustomTransport) | +| [`ListBuckets`](#ListBuckets) | [`CopyObject`](#CopyObject) | [`CopyObject`](#CopyObject) | [`PresignedPostPolicy`](#PresignedPostPolicy) | [`SetBucketNotification`](#SetBucketNotification) | [`TraceOn`](#TraceOn) | +| [`BucketExists`](#BucketExists) | [`StatObject`](#StatObject) | [`StatObject`](#StatObject) | | [`GetBucketNotification`](#GetBucketNotification) | [`TraceOff`](#TraceOff) | +| [`RemoveBucket`](#RemoveBucket) | [`RemoveObject`](#RemoveObject) | | | [`RemoveAllBucketNotification`](#RemoveAllBucketNotification) | [`SetS3TransferAccelerate`](#SetS3TransferAccelerate) | +| [`ListObjects`](#ListObjects) | [`RemoveObjects`](#RemoveObjects) | | | [`ListenBucketNotification`](#ListenBucketNotification) | | +| [`ListObjectsV2`](#ListObjectsV2) | [`RemoveIncompleteUpload`](#RemoveIncompleteUpload) | | | [`SetBucketLifecycle`](#SetBucketLifecycle) | | +| [`ListIncompleteUploads`](#ListIncompleteUploads) | [`FPutObject`](#FPutObject) | [`FPutObject`](#FPutObject) | | [`GetBucketLifecycle`](#GetBucketLifecycle) | | +| [`SetBucketTagging`](#SetBucketTagging) | [`FGetObject`](#FGetObject) | [`FGetObject`](#FGetObject) | | [`SetObjectLockConfig`](#SetObjectLockConfig) | | +| [`GetBucketTagging`](#GetBucketTagging) | [`ComposeObject`](#ComposeObject) | [`ComposeObject`](#ComposeObject) | | [`GetObjectLockConfig`](#GetObjectLockConfig) | | +| [`DeleteBucketTagging`](#DeleteBucketTagging) | [`NewSourceInfo`](#NewSourceInfo) | [`NewSourceInfo`](#NewSourceInfo) | | [`EnableVersioning`](#EnableVersioning) | | +| | [`NewDestinationInfo`](#NewDestinationInfo) | [`NewDestinationInfo`](#NewDestinationInfo) | | [`DisableVersioning`](#DisableVersioning) | | +| | [`PutObjectWithContext`](#PutObjectWithContext) | [`PutObjectWithContext`](#PutObjectWithContext) | | [`GetBucketVersioning`](#GetBucketVersioning) | | +| | [`GetObjectWithContext`](#GetObjectWithContext) | [`GetObjectWithContext`](#GetObjectWithContext) | | [`SetBucketEncryption`](#SetBucketEncryption) | | +| | [`FPutObjectWithContext`](#FPutObjectWithContext) | [`FPutObjectWithContext`](#FPutObjectWithContext) | | [`GetBucketEncryption`](#GetBucketEncryption) | | +| | [`FGetObjectWithContext`](#FGetObjectWithContext) | [`FGetObjectWithContext`](#FGetObjectWithContext) | | [`DeleteBucketEncryption`](#DeleteBucketEncryption) | | +| | [`RemoveObjectsWithContext`](#RemoveObjectsWithContext) | | | | | +| | [`RemoveObjectsWithOptions`](#RemoveObjectsWithOptions) | | | | | +| | [`RemoveObjectsWithOptionsContext`](#RemoveObjectsWithOptionsContext) | | | | | +| | [`RemoveObjectWithOptions`](#RemoveObjectWithOptions) | | | | | +| | [`PutObjectRetention`](#PutObjectRetention) | | | | | +| | [`GetObjectRetention`](#GetObjectRetention) | | | | | +| | [`PutObjectLegalHold`](#PutObjectLegalHold) | | | | | +| | [`GetObjectLegalHold`](#GetObjectLegalHold) | | | | | +| | [`SelectObjectContent`](#SelectObjectContent) | | | | | +| | [`PutObjectTagging`](#PutObjectTagging) | | | | | +| | [`PutObjectTaggingWithContext`](#PutObjectTaggingWithContext) | | | | | +| | [`GetObjectTagging`](#GetObjectTagging) | | | | | +| | [`GetObjectTaggingWithContext`](#GetObjectTaggingWithContext) | | | | | +| | [`RemoveObjectTagging`](#RemoveObjectTagging) | | | | | +| | [`RemoveObjectTaggingWithContext`](#RemoveObjectTaggingWithContext) | | | | | ## 1. Constructor @@ -433,6 +433,78 @@ for multiPartObject := range multiPartObjectCh { } ``` + +### SetBucketTagging(bucketName string, tags *tags.Tags) error +Sets tags to a bucket. + + +__Parameters__ +| Param | Type | Description | +|:-------------|:-------------|:-------------------| +| `bucketName` | _string_ | Name of the bucket | +| `tags` | _*tags.Tags_ | Bucket tags | + +__Example__ +```go +// Create tags from a map. +tags, err := tags.NewTags(map[string]string{ + "Tag1": "Value1", + "Tag2": "Value2", +}, false) +if err != nil { + log.Fatalln(err) +} + +err = minioClient.SetBucketTagging("my-bucketname", tags) +if err != nil { + log.Fatalln(err) +} +``` + + +### GetBucketTagging(bucketName string) (*tags.Tags, error) +Gets tags of a bucket. + + +__Parameters__ +| Param | Type | Description | +|:-------------|:-------------|:-------------------| +| `bucketName` | _string_ | Name of the bucket | + +__Return Value__ + +| Param | Type | Description | +|:-------|:-------------|:------------| +| `tags` | _*tags.Tags_ | Bucket tags | + +__Example__ +```go +tags, err := minioClient.GetBucketTagging("my-bucketname") +if err != nil { + log.Fatalln(err) +} + +fmt.Printf("Fetched Object Tags: %v\n", tags) +``` + + +### DeleteBucketTagging(bucketName string) error +Deletes all tags of a bucket. + + +__Parameters__ +| Param | Type | Description | +|:-------------|:-------------|:-------------------| +| `bucketName` | _string_ | Name of the bucket | + +__Example__ +```go +err := minioClient.DeleteBucketTagging("my-bucketname") +if err != nil { + log.Fatalln(err) +} +``` + ## 3. Object operations diff --git a/examples/s3/getbuckettagging.go b/examples/s3/getbuckettagging.go new file mode 100644 index 000000000..27d87092a --- /dev/null +++ b/examples/s3/getbuckettagging.go @@ -0,0 +1,47 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + "log" + + minio "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and + // my-objectname are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + tags, err := s3Client.GetBucketTagging("my-bucketname") + if err != nil { + log.Fatalln(err) + } + fmt.Printf("Fetched Object Tags: %s", tags) +} diff --git a/examples/s3/removebuckettagging.go b/examples/s3/removebuckettagging.go new file mode 100644 index 000000000..76780aa35 --- /dev/null +++ b/examples/s3/removebuckettagging.go @@ -0,0 +1,45 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + minio "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and + // my-objectname are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + err = s3Client.DeleteBucketTagging("my-bucketname") + if err != nil { + log.Fatalln(err) + } +} diff --git a/examples/s3/setbuckettagging.go b/examples/s3/setbuckettagging.go new file mode 100644 index 000000000..3f6f9c856 --- /dev/null +++ b/examples/s3/setbuckettagging.go @@ -0,0 +1,55 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + minio "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v6/pkg/tags" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and + // my-objectname are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + tags, err := tags.NewTags(map[string]string{ + "Tag1": "Value1", + "Tag2": "Value2", + }, false) + if err != nil { + log.Fatalln(err) + } + + err = s3Client.SetBucketTagging("my-bucketname", tags) + if err != nil { + log.Fatalln(err) + } +} From 243245c80be6de72365625fbeae7b5980905fc0c Mon Sep 17 00:00:00 2001 From: Bala FA Date: Thu, 30 Apr 2020 22:28:31 +0000 Subject: [PATCH 149/215] fix: Accept 204 as success (#1281) --- api-bucket-tagging.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-bucket-tagging.go b/api-bucket-tagging.go index 0ebc4b27f..ab1828b77 100644 --- a/api-bucket-tagging.go +++ b/api-bucket-tagging.go @@ -107,7 +107,7 @@ func (c Client) SetBucketTaggingWithContext(ctx context.Context, bucketName stri if err != nil { return err } - if resp.StatusCode != http.StatusOK { + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { return httpRespToErrorResponse(resp, bucketName, "") } return nil From a81c8c13cc3f06539692c3be25fff218da98f4c7 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 1 May 2020 18:32:57 -0700 Subject: [PATCH 150/215] honor doneCh for listenBucketNotification (#1282) --- api-notification.go | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/api-notification.go b/api-notification.go index cae3b14e4..24ee121e3 100644 --- a/api-notification.go +++ b/api-notification.go @@ -146,16 +146,22 @@ func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, even // Validate the bucket name. if err := s3utils.CheckValidBucketName(bucketName); err != nil { - notificationInfoCh <- NotificationInfo{ + select { + case notificationInfoCh <- NotificationInfo{ Err: err, + }: + case <-doneCh: } return } // Check ARN partition to verify if listening bucket is supported if s3utils.IsAmazonEndpoint(*c.endpointURL) || s3utils.IsGoogleEndpoint(*c.endpointURL) { - notificationInfoCh <- NotificationInfo{ + select { + case notificationInfoCh <- NotificationInfo{ Err: ErrAPINotSupported("Listening for bucket notification is specific only to `minio` server endpoints"), + }: + case <-doneCh: } return } @@ -176,14 +182,17 @@ func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, even // Wait on the jitter retry loop. for range c.newRetryTimerContinous(time.Second, time.Second*30, MaxJitter, retryDoneCh) { // Execute GET on bucket to list objects. - resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + resp, err := c.executeMethod(context.Background(), http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, }) if err != nil { - notificationInfoCh <- NotificationInfo{ + select { + case notificationInfoCh <- NotificationInfo{ Err: err, + }: + case <-doneCh: } return } @@ -191,8 +200,11 @@ func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, even // Validate http response, upon error return quickly. if resp.StatusCode != http.StatusOK { errResponse := httpRespToErrorResponse(resp, bucketName, "") - notificationInfoCh <- NotificationInfo{ + select { + case notificationInfoCh <- NotificationInfo{ Err: errResponse, + }: + case <-doneCh: } return } @@ -211,8 +223,12 @@ func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, even if err = json.Unmarshal(bio.Bytes(), ¬ificationInfo); err != nil { // Unexpected error during json unmarshal, send // the error to caller for actionable as needed. - notificationInfoCh <- NotificationInfo{ + select { + case notificationInfoCh <- NotificationInfo{ Err: err, + }: + case <-doneCh: + return } closeResponse(resp) continue @@ -225,13 +241,20 @@ func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, even return } } + if err = bio.Err(); err != nil { - notificationInfoCh <- NotificationInfo{ + select { + case notificationInfoCh <- NotificationInfo{ Err: err, + }: + case <-doneCh: + return } } + // Close current connection before looping further. closeResponse(resp) + } }(notificationInfoCh) From 79f99bc54ad036fd2ef9166149e9a8de33920edc Mon Sep 17 00:00:00 2001 From: Chad Denyar Date: Fri, 8 May 2020 19:22:39 -0400 Subject: [PATCH 151/215] support for new af-south-1 region (#1285) --- s3-endpoints.go | 1 + 1 file changed, 1 insertion(+) diff --git a/s3-endpoints.go b/s3-endpoints.go index 4a8879fd5..37c826dd4 100644 --- a/s3-endpoints.go +++ b/s3-endpoints.go @@ -36,6 +36,7 @@ var awsS3EndpointMap = map[string]string{ "ap-northeast-1": "s3.dualstack.ap-northeast-1.amazonaws.com", "ap-northeast-2": "s3.dualstack.ap-northeast-2.amazonaws.com", "ap-northeast-3": "s3.dualstack.ap-northeast-3.amazonaws.com", + "af-south-1": "s3.dualstack.af-south-1.amazonaws.com", "me-south-1": "s3.dualstack.me-south-1.amazonaws.com", "sa-east-1": "s3.dualstack.sa-east-1.amazonaws.com", "us-gov-west-1": "s3.dualstack.us-gov-west-1.amazonaws.com", From d3e246d5a45356ca2dc883d9ca0cfa0ffbce5177 Mon Sep 17 00:00:00 2001 From: Yefry Figueroa Date: Thu, 21 May 2020 10:30:15 +0200 Subject: [PATCH 152/215] Add new Milan AWS Region (#1286) include the support for new eu-south-1 --- s3-endpoints.go | 1 + 1 file changed, 1 insertion(+) diff --git a/s3-endpoints.go b/s3-endpoints.go index 37c826dd4..125d86289 100644 --- a/s3-endpoints.go +++ b/s3-endpoints.go @@ -29,6 +29,7 @@ var awsS3EndpointMap = map[string]string{ "eu-west-3": "s3.dualstack.eu-west-3.amazonaws.com", "eu-central-1": "s3.dualstack.eu-central-1.amazonaws.com", "eu-north-1": "s3.dualstack.eu-north-1.amazonaws.com", + "eu-south-1": "s3.dualstack.eu-south-1.amazonaws.com", "ap-east-1": "s3.dualstack.ap-east-1.amazonaws.com", "ap-south-1": "s3.dualstack.ap-south-1.amazonaws.com", "ap-southeast-1": "s3.dualstack.ap-southeast-1.amazonaws.com", From e9bc14bbccf9e054484aa7e6ef4bdbc0e6a7e11e Mon Sep 17 00:00:00 2001 From: P R <25353498+BigUstad@users.noreply.github.com> Date: Thu, 21 May 2020 17:50:53 -0700 Subject: [PATCH 153/215] core: Modify Putobject to use PutObjectOptions (#1290) Replacing ObjectOptions, used in gateway and other places --- core.go | 28 ++------------ core_test.go | 34 +++++++++++------ functional_tests.go | 90 ++++++++++++++++++++++++++++++--------------- 3 files changed, 87 insertions(+), 65 deletions(-) diff --git a/core.go b/core.go index 277210a8d..6d1fddb2e 100644 --- a/core.go +++ b/core.go @@ -21,7 +21,6 @@ import ( "context" "io" "net/http" - "strings" "github.com/minio/minio-go/v6/pkg/encrypt" ) @@ -85,34 +84,13 @@ func (c Core) CopyObjectPart(srcBucket, srcObject, destBucket, destObject string } // PutObjectWithContext - Upload object. Uploads using single PUT call. -func (c Core) PutObjectWithContext(ctx context.Context, bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, metadata map[string]string, sse encrypt.ServerSide) (ObjectInfo, error) { - opts := PutObjectOptions{} - m := make(map[string]string) - for k, v := range metadata { - if strings.ToLower(k) == "content-encoding" { - opts.ContentEncoding = v - } else if strings.ToLower(k) == "content-disposition" { - opts.ContentDisposition = v - } else if strings.ToLower(k) == "content-language" { - opts.ContentLanguage = v - } else if strings.ToLower(k) == "content-type" { - opts.ContentType = v - } else if strings.ToLower(k) == "cache-control" { - opts.CacheControl = v - } else if strings.EqualFold(k, amzWebsiteRedirectLocation) { - opts.WebsiteRedirectLocation = v - } else { - m[k] = metadata[k] - } - } - opts.UserMetadata = m - opts.ServerSideEncryption = sse +func (c Core) PutObjectWithContext(ctx context.Context, bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, opts PutObjectOptions) (ObjectInfo, error) { return c.putObjectDo(ctx, bucket, object, data, md5Base64, sha256Hex, size, opts) } // PutObject - Upload object. Uploads using single PUT call. -func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, metadata map[string]string, sse encrypt.ServerSide) (ObjectInfo, error) { - return c.PutObjectWithContext(context.Background(), bucket, object, data, size, md5Base64, sha256Hex, metadata, sse) +func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, opts PutObjectOptions) (ObjectInfo, error) { + return c.PutObjectWithContext(context.Background(), bucket, object, data, size, md5Base64, sha256Hex, opts) } // NewMultipartUpload - Initiates new multipart upload and returns the new uploadID. diff --git a/core_test.go b/core_test.go index e335dd43e..312efd9e3 100644 --- a/core_test.go +++ b/core_test.go @@ -434,9 +434,13 @@ func TestCoreCopyObject(t *testing.T) { // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", map[string]string{ - "Content-Type": "binary/octet-stream", - }, nil) + + putopts := PutObjectOptions{ + UserMetadata: map[string]string{ + "Content-Type": "binary/octet-stream", + }, + } + objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", putopts) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } @@ -546,12 +550,15 @@ func TestCoreCopyObjectPart(t *testing.T) { // Make a buffer with 5MB of data buf := bytes.Repeat([]byte("abcde"), 1024*1024) - + metadata := map[string]string{ + "Content-Type": "binary/octet-stream", + } + putopts := PutObjectOptions{ + UserMetadata: metadata, + } // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", map[string]string{ - "Content-Type": "binary/octet-stream", - }, nil) + objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", putopts) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } @@ -697,13 +704,15 @@ func TestCorePutObject(t *testing.T) { objectContentType := "binary/octet-stream" metadata := make(map[string]string) metadata["Content-Type"] = objectContentType - - objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "1B2M2Y8AsgTpgAmY7PhCfg==", "", metadata, nil) + putopts := PutObjectOptions{ + UserMetadata: metadata, + } + objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "1B2M2Y8AsgTpgAmY7PhCfg==", "", putopts) if err == nil { t.Fatal("Error expected: error, got: nil(success)") } - objInfo, err = c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", metadata, nil) + objInfo, err = c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", putopts) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } @@ -777,9 +786,12 @@ func TestCoreGetObjectMetadata(t *testing.T) { metadata := map[string]string{ "X-Amz-Meta-Key-1": "Val-1", } + putopts := PutObjectOptions{ + UserMetadata: metadata, + } _, err = core.PutObject(bucketName, "my-objectname", - bytes.NewReader([]byte("hello")), 5, "", "", metadata, nil) + bytes.NewReader([]byte("hello")), 5, "", "", putopts) if err != nil { log.Fatalln(err) } diff --git a/functional_tests.go b/functional_tests.go index 70b757156..dcdf30364 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -6966,10 +6966,14 @@ func testSSECEncryptedToSSECCopyObjectPart() { objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") password := "correct horse battery staple" srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) - - objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", map[string]string{ + putmetadata := map[string]string{ "Content-Type": "binary/octet-stream", - }, srcencryption) + } + opts := minio.PutObjectOptions{ + UserMetadata: putmetadata, + ServerSideEncryption: srcencryption, + } + objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } @@ -7118,9 +7122,13 @@ func testSSECEncryptedToUnencryptedCopyPart() { password := "correct horse battery staple" srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) - objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", map[string]string{ - "Content-Type": "binary/octet-stream", - }, srcencryption) + opts := minio.PutObjectOptions{ + UserMetadata: map[string]string{ + "Content-Type": "binary/octet-stream", + }, + ServerSideEncryption: srcencryption, + } + objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } @@ -7267,10 +7275,15 @@ func testSSECEncryptedToSSES3CopyObjectPart() { objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") password := "correct horse battery staple" srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) - - objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", map[string]string{ + putmetadata := map[string]string{ "Content-Type": "binary/octet-stream", - }, srcencryption) + } + opts := minio.PutObjectOptions{ + UserMetadata: putmetadata, + ServerSideEncryption: srcencryption, + } + + objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } @@ -7418,10 +7431,13 @@ func testUnencryptedToSSECCopyObjectPart() { // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") password := "correct horse battery staple" - - objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", map[string]string{ + putmetadata := map[string]string{ "Content-Type": "binary/octet-stream", - }, nil) + } + opts := minio.PutObjectOptions{ + UserMetadata: putmetadata, + } + objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } @@ -7566,10 +7582,13 @@ func testUnencryptedToUnencryptedCopyPart() { // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - - objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", map[string]string{ + putmetadata := map[string]string{ "Content-Type": "binary/octet-stream", - }, nil) + } + opts := minio.PutObjectOptions{ + UserMetadata: putmetadata, + } + objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } @@ -7712,10 +7731,12 @@ func testUnencryptedToSSES3CopyObjectPart() { // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - - objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", map[string]string{ - "Content-Type": "binary/octet-stream", - }, nil) + opts := minio.PutObjectOptions{ + UserMetadata: map[string]string{ + "Content-Type": "binary/octet-stream", + }, + } + objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } @@ -7863,9 +7884,13 @@ func testSSES3EncryptedToSSECCopyObjectPart() { objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") password := "correct horse battery staple" srcEncryption := encrypt.NewSSE() - objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", map[string]string{ - "Content-Type": "binary/octet-stream", - }, srcEncryption) + opts := minio.PutObjectOptions{ + UserMetadata: map[string]string{ + "Content-Type": "binary/octet-stream", + }, + ServerSideEncryption: srcEncryption, + } + objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } @@ -8011,10 +8036,13 @@ func testSSES3EncryptedToUnencryptedCopyPart() { // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") srcEncryption := encrypt.NewSSE() - - objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", map[string]string{ - "Content-Type": "binary/octet-stream", - }, srcEncryption) + opts := minio.PutObjectOptions{ + UserMetadata: map[string]string{ + "Content-Type": "binary/octet-stream", + }, + ServerSideEncryption: srcEncryption, + } + objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } @@ -8158,10 +8186,14 @@ func testSSES3EncryptedToSSES3CopyObjectPart() { // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") srcEncryption := encrypt.NewSSE() + opts := minio.PutObjectOptions{ + UserMetadata: map[string]string{ + "Content-Type": "binary/octet-stream", + }, + ServerSideEncryption: srcEncryption, + } - objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", map[string]string{ - "Content-Type": "binary/octet-stream", - }, srcEncryption) + objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } From 44a5f2e3b76b63385ed69541a160b83260e0647c Mon Sep 17 00:00:00 2001 From: kannappanr <30541348+kannappanr@users.noreply.github.com> Date: Fri, 22 May 2020 09:49:46 -0700 Subject: [PATCH 154/215] Add tagging count to known header (#1291) --- utils.go | 1 + 1 file changed, 1 insertion(+) diff --git a/utils.go b/utils.go index 685e98c70..349c86e28 100644 --- a/utils.go +++ b/utils.go @@ -175,6 +175,7 @@ func extractObjMetadata(header http.Header) http.Header { "X-Amz-Object-Lock-Legal-Hold", "X-Amz-Website-Redirect-Location", "X-Amz-Server-Side-Encryption", + "X-Amz-Tagging-Count", "X-Amz-Meta-", // Add new headers to be preserved. // if you add new headers here, please extend From 360c4f11a4978da7a1d2ef46a0b47f888b9743ae Mon Sep 17 00:00:00 2001 From: kannappanr <30541348+kannappanr@users.noreply.github.com> Date: Tue, 26 May 2020 09:32:25 -0700 Subject: [PATCH 155/215] retry makebucket on AuthorizationHeaderMalformed code and empty location (#1292) If makebucket was called with location not set and the server returns AuthorizationHeaderMalformed with the correct region, use that and retry makebucket again. --- api-put-bucket.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/api-put-bucket.go b/api-put-bucket.go index 8d9671b1e..9d2486ad0 100644 --- a/api-put-bucket.go +++ b/api-put-bucket.go @@ -51,6 +51,21 @@ type ServerSideEncryptionConfiguration struct { /// Bucket operations func (c Client) makeBucket(ctx context.Context, bucketName string, location string, objectLockEnabled bool) (err error) { + // Validate the input arguments. + if err := s3utils.CheckValidBucketNameStrict(bucketName); err != nil { + return err + } + + err = c.doMakeBucket(ctx, bucketName, location, objectLockEnabled) + if err != nil && (location == "" || location == "us-east-1") { + if resp, ok := err.(ErrorResponse); ok && resp.Code == "AuthorizationHeaderMalformed" && resp.Region != "" { + err = c.doMakeBucket(ctx, bucketName, resp.Region, objectLockEnabled) + } + } + return err +} + +func (c Client) doMakeBucket(ctx context.Context, bucketName string, location string, objectLockEnabled bool) (err error) { defer func() { // Save the location into cache on a successful makeBucket response. if err == nil { @@ -58,11 +73,6 @@ func (c Client) makeBucket(ctx context.Context, bucketName string, location stri } }() - // Validate the input arguments. - if err := s3utils.CheckValidBucketNameStrict(bucketName); err != nil { - return err - } - // If location is empty, treat is a default region 'us-east-1'. if location == "" { location = "us-east-1" From 6a226f4714bbadae829674c06ce27d77836cb457 Mon Sep 17 00:00:00 2001 From: kannappanr <30541348+kannappanr@users.noreply.github.com> Date: Thu, 28 May 2020 10:34:26 -0700 Subject: [PATCH 156/215] retry only if response region doesn't match cached region (#1293) --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index f9bdbb9cd..cbc5f8f41 100644 --- a/api.go +++ b/api.go @@ -683,7 +683,7 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque // handle this appropriately. if metadata.bucketName != "" { // Gather Cached location only if bucketName is present. - if _, cachedOk := c.bucketLocCache.Get(metadata.bucketName); cachedOk { + if location, cachedOk := c.bucketLocCache.Get(metadata.bucketName); cachedOk && location != errResponse.Region { c.bucketLocCache.Set(metadata.bucketName, errResponse.Region) continue // Retry. } From dc7b291f154ffce687087a72e5c8fa82a39d00ab Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Thu, 28 May 2020 21:47:34 +0000 Subject: [PATCH 157/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index cbc5f8f41..0e356fef2 100644 --- a/api.go +++ b/api.go @@ -104,7 +104,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.56" + libraryVersion = "v6.0.57" ) // User Agent should always following the below style. From be8a97771e7907561dc974912f45f6b9b40ba118 Mon Sep 17 00:00:00 2001 From: kannappanr <30541348+kannappanr@users.noreply.github.com> Date: Fri, 29 May 2020 23:30:53 -0700 Subject: [PATCH 158/215] Treat x-amz-metadata-directive as a standard header (#1296) Fixes #1295 --- utils.go | 1 + 1 file changed, 1 insertion(+) diff --git a/utils.go b/utils.go index 349c86e28..b5cb8c537 100644 --- a/utils.go +++ b/utils.go @@ -325,6 +325,7 @@ var supportedHeaders = []string{ "content-language", "x-amz-website-redirect-location", "x-amz-object-lock-mode", + "x-amz-metadata-directive", "x-amz-object-lock-retain-until-date", "expires", // Add more supported headers here. From bc3a8c591b871215c94712c7563036369dd8c057 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Tue, 2 Jun 2020 14:54:09 -0700 Subject: [PATCH 159/215] Add custom MD5/SHA256 hasher option (#1283) --- api-put-object-multipart.go | 1 + api-put-object-streaming.go | 8 ++-- api-put-object.go | 4 +- api.go | 96 +++++++++++++++++++++++++------------ core_test.go | 3 +- go.mod | 1 + go.sum | 7 +++ utils.go | 43 +++++++++++++++-- 8 files changed, 122 insertions(+), 41 deletions(-) diff --git a/api-put-object-multipart.go b/api-put-object-multipart.go index e5bd37d5d..25ae63333 100644 --- a/api-put-object-multipart.go +++ b/api-put-object-multipart.go @@ -118,6 +118,7 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje for k, v := range hashAlgos { v.Write(buf[:length]) hashSums[k] = v.Sum(nil) + v.Close() } // Update progress reader appropriately to the latest offset diff --git a/api-put-object-streaming.go b/api-put-object-streaming.go index 4da4174cd..ceb53421b 100644 --- a/api-put-object-streaming.go +++ b/api-put-object-streaming.go @@ -20,7 +20,6 @@ package minio import ( "bytes" "context" - "crypto/md5" "encoding/base64" "fmt" "io" @@ -287,9 +286,11 @@ func (c Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, bu return 0, rerr } // Calculate md5sum. - hash := md5.New() + hash := c.md5Hasher() hash.Write(buf[:length]) md5Base64 = base64.StdEncoding.EncodeToString(hash.Sum(nil)) + hash.Close() + // Update progress reader appropriately to the latest offset // as we read from the source. hookReader = newHook(bytes.NewReader(buf[:length]), opts.Progress) @@ -393,10 +394,11 @@ func (c Client) putObject(ctx context.Context, bucketName, objectName string, re } // Calculate md5sum. - hash := md5.New() + hash := c.md5Hasher() hash.Write(buf[:length]) md5Base64 = base64.StdEncoding.EncodeToString(hash.Sum(nil)) reader = bytes.NewReader(buf[:length]) + hash.Close() } // Update progress reader appropriately to the latest offset as we diff --git a/api-put-object.go b/api-put-object.go index eabe788da..d4188d580 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -20,7 +20,6 @@ package minio import ( "bytes" "context" - "crypto/md5" "encoding/base64" "errors" "fmt" @@ -267,9 +266,10 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName var md5Base64 string if opts.SendContentMd5 { // Calculate md5sum. - hash := md5.New() + hash := c.md5Hasher() hash.Write(buf[:length]) md5Base64 = base64.StdEncoding.EncodeToString(hash.Sum(nil)) + hash.Close() } // Update progress reader appropriately to the latest offset diff --git a/api.go b/api.go index 0e356fef2..22778dafa 100644 --- a/api.go +++ b/api.go @@ -20,10 +20,8 @@ package minio import ( "bytes" "context" - "crypto/md5" "errors" "fmt" - "hash" "io" "io/ioutil" "math/rand" @@ -38,13 +36,11 @@ import ( "sync" "time" - "github.com/minio/sha256-simd" - - "golang.org/x/net/publicsuffix" - + md5simd "github.com/minio/md5-simd" "github.com/minio/minio-go/v6/pkg/credentials" "github.com/minio/minio-go/v6/pkg/s3utils" "github.com/minio/minio-go/v6/pkg/signer" + "golang.org/x/net/publicsuffix" ) // Client implements Amazon S3 compatible methods. @@ -90,6 +86,10 @@ type Client struct { // lookup indicates type of url lookup supported by server. If not specified, // default to Auto. lookup BucketLookupType + + // Factory for MD5 hash functions. + md5Hasher func() md5simd.Hasher + sha256Hasher func() md5simd.Hasher } // Options for New method @@ -98,7 +98,10 @@ type Options struct { Secure bool Region string BucketLookup BucketLookupType - // Add future fields here + + // Custom hash routines. Leave nil to use standard. + CustomMD5 func() md5simd.Hasher + CustomSHA256 func() md5simd.Hasher } // Global constants. @@ -129,8 +132,11 @@ const ( // NewV2 - instantiate minio client with Amazon S3 signature version // '2' compatibility. func NewV2(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { - creds := credentials.NewStaticV2(accessKeyID, secretAccessKey, "") - clnt, err := privateNew(endpoint, creds, secure, "", BucketLookupAuto) + clnt, err := privateNew(endpoint, Options{ + Creds: credentials.NewStaticV2(accessKeyID, secretAccessKey, ""), + Secure: secure, + BucketLookup: BucketLookupAuto, + }) if err != nil { return nil, err } @@ -141,8 +147,11 @@ func NewV2(endpoint string, accessKeyID, secretAccessKey string, secure bool) (* // NewV4 - instantiate minio client with Amazon S3 signature version // '4' compatibility. func NewV4(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { - creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "") - clnt, err := privateNew(endpoint, creds, secure, "", BucketLookupAuto) + clnt, err := privateNew(endpoint, Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: secure, + BucketLookup: BucketLookupAuto, + }) if err != nil { return nil, err } @@ -152,8 +161,11 @@ func NewV4(endpoint string, accessKeyID, secretAccessKey string, secure bool) (* // New - instantiate minio client, adds automatic verification of signature. func New(endpoint, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { - creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "") - clnt, err := privateNew(endpoint, creds, secure, "", BucketLookupAuto) + clnt, err := privateNew(endpoint, Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: secure, + BucketLookup: BucketLookupAuto, + }) if err != nil { return nil, err } @@ -172,7 +184,12 @@ func New(endpoint, accessKeyID, secretAccessKey string, secure bool) (*Client, e // for retrieving credentials from various credentials provider such as // IAM, File, Env etc. func NewWithCredentials(endpoint string, creds *credentials.Credentials, secure bool, region string) (*Client, error) { - return privateNew(endpoint, creds, secure, region, BucketLookupAuto) + return privateNew(endpoint, Options{ + Creds: creds, + Secure: secure, + BucketLookup: BucketLookupAuto, + Region: region, + }) } // NewWithRegion - instantiate minio client, with region configured. Unlike New(), @@ -180,12 +197,20 @@ func NewWithCredentials(endpoint string, creds *credentials.Credentials, secure // Use this function when if your application deals with single region. func NewWithRegion(endpoint, accessKeyID, secretAccessKey string, secure bool, region string) (*Client, error) { creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "") - return privateNew(endpoint, creds, secure, region, BucketLookupAuto) + return privateNew(endpoint, Options{ + Creds: creds, + Secure: secure, + Region: region, + BucketLookup: BucketLookupAuto, + }) } // NewWithOptions - instantiate minio client with options func NewWithOptions(endpoint string, opts *Options) (*Client, error) { - return privateNew(endpoint, opts.Creds, opts.Secure, opts.Region, opts.BucketLookup) + if opts == nil { + return nil, errors.New("no options provided") + } + return privateNew(endpoint, *opts) } // EndpointURL returns the URL of the S3 endpoint. @@ -277,9 +302,9 @@ func (c *Client) redirectHeaders(req *http.Request, via []*http.Request) error { return nil } -func privateNew(endpoint string, creds *credentials.Credentials, secure bool, region string, lookup BucketLookupType) (*Client, error) { +func privateNew(endpoint string, opts Options) (*Client, error) { // construct endpoint. - endpointURL, err := getEndpointURL(endpoint, secure) + endpointURL, err := getEndpointURL(endpoint, opts.Secure) if err != nil { return nil, err } @@ -295,15 +320,15 @@ func privateNew(endpoint string, creds *credentials.Credentials, secure bool, re clnt := new(Client) // Save the credentials. - clnt.credsProvider = creds + clnt.credsProvider = opts.Creds // Remember whether we are using https or not - clnt.secure = secure + clnt.secure = opts.Secure // Save endpoint URL, user agent for future uses. clnt.endpointURL = endpointURL - transport, err := DefaultTransport(secure) + transport, err := DefaultTransport(opts.Secure) if err != nil { return nil, err } @@ -316,10 +341,10 @@ func privateNew(endpoint string, creds *credentials.Credentials, secure bool, re } // Sets custom region, if region is empty bucket location cache is used automatically. - if region == "" { - region = s3utils.GetRegionFromURL(*clnt.endpointURL) + if opts.Region == "" { + opts.Region = s3utils.GetRegionFromURL(*clnt.endpointURL) } - clnt.region = region + clnt.region = opts.Region // Instantiate bucket location cache. clnt.bucketLocCache = newBucketLocationCache() @@ -327,9 +352,18 @@ func privateNew(endpoint string, creds *credentials.Credentials, secure bool, re // Introduce a new locked random seed. clnt.random = rand.New(&lockedRandSource{src: rand.NewSource(time.Now().UTC().UnixNano())}) + // Add default md5 hasher. + clnt.md5Hasher = opts.CustomMD5 + clnt.sha256Hasher = opts.CustomSHA256 + if clnt.md5Hasher == nil { + clnt.md5Hasher = newMd5Hasher + } + if clnt.sha256Hasher == nil { + clnt.sha256Hasher = newSHA256Hasher + } // Sets bucket lookup style, whether server accepts DNS or Path lookup. Default is Auto - determined // by the SDK. When Auto is specified, DNS lookup is used for Amazon/Google cloud endpoints and Path for all other endpoints. - clnt.lookup = lookup + clnt.lookup = opts.BucketLookup // Return. return clnt, nil } @@ -413,22 +447,22 @@ func (c *Client) SetS3TransferAccelerate(accelerateEndpoint string) { // - For signature v4 request if the connection is insecure compute only sha256. // - For signature v4 request if the connection is secure compute only md5. // - For anonymous request compute md5. -func (c *Client) hashMaterials(isMd5Requested bool) (hashAlgos map[string]hash.Hash, hashSums map[string][]byte) { +func (c *Client) hashMaterials(isMd5Requested bool) (hashAlgos map[string]md5simd.Hasher, hashSums map[string][]byte) { hashSums = make(map[string][]byte) - hashAlgos = make(map[string]hash.Hash) + hashAlgos = make(map[string]md5simd.Hasher) if c.overrideSignerType.IsV4() { if c.secure { - hashAlgos["md5"] = md5.New() + hashAlgos["md5"] = c.md5Hasher() } else { - hashAlgos["sha256"] = sha256.New() + hashAlgos["sha256"] = c.sha256Hasher() } } else { if c.overrideSignerType.IsAnonymous() { - hashAlgos["md5"] = md5.New() + hashAlgos["md5"] = c.md5Hasher() } } if isMd5Requested { - hashAlgos["md5"] = md5.New() + hashAlgos["md5"] = c.md5Hasher() } return hashAlgos, hashSums } diff --git a/core_test.go b/core_test.go index 312efd9e3..ada24605c 100644 --- a/core_test.go +++ b/core_test.go @@ -678,7 +678,8 @@ func TestCorePutObject(t *testing.T) { mustParseBool(os.Getenv(enableSecurity)), ) if err != nil { - t.Fatal("Error:", err) + t.Error("Error:", err) + return } // Enable tracing, write to stderr. diff --git a/go.mod b/go.mod index fe43a735e..03497bccf 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.12 require ( github.com/dustin/go-humanize v1.0.0 // indirect github.com/json-iterator/go v1.1.9 + github.com/minio/md5-simd v1.1.0 github.com/minio/sha256-simd v0.1.1 github.com/mitchellh/go-homedir v1.1.0 github.com/sirupsen/logrus v1.5.0 // indirect diff --git a/go.sum b/go.sum index 7461ddbf1..f158252bb 100644 --- a/go.sum +++ b/go.sum @@ -10,7 +10,13 @@ github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGn github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs= +github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/minio/md5-simd v1.0.1 h1:tj/FH8APTKxIkOGUX2YGAVJVXXC3AJ5T2SkHoT/dUFI= +github.com/minio/md5-simd v1.0.1/go.mod h1:EhdyA+Dr0guvfyc8d6yrgs9YzHGfaI+YIjA6gt/7mJk= +github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= +github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -21,6 +27,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLD github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q= github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= diff --git a/utils.go b/utils.go index b5cb8c537..0214644e4 100644 --- a/utils.go +++ b/utils.go @@ -22,6 +22,7 @@ import ( "encoding/base64" "encoding/hex" "encoding/xml" + "hash" "io" "io/ioutil" "net" @@ -30,11 +31,12 @@ import ( "regexp" "strconv" "strings" + "sync" "time" - "github.com/minio/sha256-simd" - + md5simd "github.com/minio/md5-simd" "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/sha256-simd" ) func trimEtag(etag string) string { @@ -50,14 +52,16 @@ func xmlDecoder(body io.Reader, v interface{}) error { // sum256 calculate sha256sum for an input byte array, returns hex encoded. func sum256Hex(data []byte) string { - hash := sha256.New() + hash := newSHA256Hasher() + defer hash.Close() hash.Write(data) return hex.EncodeToString(hash.Sum(nil)) } // sumMD5Base64 calculate md5sum for an input byte array, returns base64 encoded. func sumMD5Base64(data []byte) string { - hash := md5.New() + hash := newMd5Hasher() + defer hash.Close() hash.Write(data) return base64.StdEncoding.EncodeToString(hash.Sum(nil)) } @@ -374,3 +378,34 @@ func isAmzHeader(headerKey string) bool { return strings.HasPrefix(key, "x-amz-meta-") || strings.HasPrefix(key, "x-amz-grant-") || key == "x-amz-acl" || isSSEHeader(headerKey) } + +var md5Pool = sync.Pool{New: func() interface{} { return md5.New() }} +var sha256Pool = sync.Pool{New: func() interface{} { return sha256.New() }} + +func newMd5Hasher() md5simd.Hasher { + return hashWrapper{Hash: md5Pool.New().(hash.Hash), isMD5: true} +} + +func newSHA256Hasher() md5simd.Hasher { + return hashWrapper{Hash: sha256Pool.New().(hash.Hash), isSHA256: true} +} + +// hashWrapper implements the md5simd.Hasher interface. +type hashWrapper struct { + hash.Hash + isMD5 bool + isSHA256 bool +} + +// Close will put the hasher back into the pool. +func (m hashWrapper) Close() { + if m.isMD5 && m.Hash != nil { + m.Reset() + md5Pool.Put(m.Hash) + } + if m.isSHA256 && m.Hash != nil { + m.Reset() + sha256Pool.Put(m.Hash) + } + m.Hash = nil +} From e50c28511b9e5ec71cb7c3118b115e83d223aa3e Mon Sep 17 00:00:00 2001 From: Justin Hutchings Date: Fri, 5 Jun 2020 21:55:32 -0700 Subject: [PATCH 160/215] Add CodeQL security scanning (#1298) --- .github/workflows/codeql.yml | 52 ++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..9771ca0f4 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,52 @@ +name: "Code scanning - action" + +on: + push: + pull_request: + schedule: + - cron: '0 19 * * 0' + +jobs: + CodeQL-Build: + + # CodeQL runs on ubuntu-latest and windows-latest + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + # We must fetch at least the immediate parents so that if this is + # a pull request then we can checkout the head. + fetch-depth: 2 + + # If this run was triggered by a pull request event, then checkout + # the head of the pull request instead of the merge commit. + - run: git checkout HEAD^2 + if: ${{ github.event_name == 'pull_request' }} + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + # Override language selection by uncommenting this and choosing your languages + # with: + # languages: go, javascript, csharp, python, cpp, java + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From 51222d26253e90224f8b4f8ba9be9b7982e3ce53 Mon Sep 17 00:00:00 2001 From: Kale Blankenship Date: Tue, 9 Jun 2020 16:17:55 -0700 Subject: [PATCH 161/215] remove usages of debug.FreeOSMemory() (#1301) This can be detrimental to an application due to: - Forces a GC on every execution and blocks until the GC is complete. - Returns OS memory immediately instead of allowing the runtime to re-use it. --- api-put-object-multipart.go | 2 -- api-put-object-streaming.go | 3 --- api-put-object.go | 2 -- 3 files changed, 7 deletions(-) diff --git a/api-put-object-multipart.go b/api-put-object-multipart.go index 25ae63333..62af16cc2 100644 --- a/api-put-object-multipart.go +++ b/api-put-object-multipart.go @@ -28,7 +28,6 @@ import ( "io/ioutil" "net/http" "net/url" - "runtime/debug" "sort" "strconv" "strings" @@ -98,7 +97,6 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje // Create a buffer. buf := make([]byte, partSize) - defer debug.FreeOSMemory() for partNumber <= totalPartsCount { // Choose hash algorithms to be calculated by hashCopyN, diff --git a/api-put-object-streaming.go b/api-put-object-streaming.go index ceb53421b..f0c5b1331 100644 --- a/api-put-object-streaming.go +++ b/api-put-object-streaming.go @@ -24,7 +24,6 @@ import ( "fmt" "io" "net/http" - "runtime/debug" "sort" "strings" @@ -262,7 +261,6 @@ func (c Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, bu // Create a buffer. buf := make([]byte, partSize) - defer debug.FreeOSMemory() // Avoid declaring variables in the for loop var md5Base64 string @@ -386,7 +384,6 @@ func (c Client) putObject(ctx context.Context, bucketName, objectName string, re if opts.SendContentMd5 { // Create a buffer. buf := make([]byte, size) - defer debug.FreeOSMemory() length, rErr := io.ReadFull(reader, buf) if rErr != nil && rErr != io.ErrUnexpectedEOF { diff --git a/api-put-object.go b/api-put-object.go index d4188d580..dbb7c8d53 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -25,7 +25,6 @@ import ( "fmt" "io" "net/http" - "runtime/debug" "sort" "time" @@ -252,7 +251,6 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName // Create a buffer. buf := make([]byte, partSize) - defer debug.FreeOSMemory() for partNumber <= totalPartsCount { length, rerr := io.ReadFull(reader, buf) From 99fad155a1d4574bcead04e83805b4557900971b Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Tue, 9 Jun 2020 23:24:31 +0000 Subject: [PATCH 162/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 22778dafa..81327de81 100644 --- a/api.go +++ b/api.go @@ -107,7 +107,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.57" + libraryVersion = "v6.0.58" ) // User Agent should always following the below style. From 3da267908ecfc26c5b9d5eb1ba44dca672f26866 Mon Sep 17 00:00:00 2001 From: ebozduman Date: Thu, 11 Jun 2020 16:37:12 -0700 Subject: [PATCH 163/215] add context support for all APIs (#1302) --- api-compose-object.go | 12 +++-- api-get-lifecycle.go | 13 ++++-- api-get-policy.go | 13 ++++-- api-list.go | 98 ++++++++++++++++++++++++++-------------- api-notification.go | 22 ++++++--- api-object-legal-hold.go | 9 +++- api-object-lock.go | 28 +++++++++--- api-object-retention.go | 9 +++- api-presigned.go | 54 ++++++++++++++++------ api-put-bucket.go | 13 ++++-- api-put-object-copy.go | 10 +++- api-remove.go | 76 +++++++++++++++++-------------- core.go | 12 ++--- 13 files changed, 246 insertions(+), 123 deletions(-) diff --git a/api-compose-object.go b/api-compose-object.go index 15e3b6f38..ef1d24440 100644 --- a/api-compose-object.go +++ b/api-compose-object.go @@ -408,16 +408,20 @@ func (c Client) uploadPartCopy(ctx context.Context, bucket, object, uploadID str return p, nil } -// ComposeObjectWithProgress - creates an object using server-side copying of +// ComposeObjectWithProgress is a wrapper for ComposeObjectWithProgressWithContext. +func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo, progress io.Reader) error { + return c.ComposeObjectWithProgressWithContext(context.Background(), dst, srcs, progress) +} + +// ComposeObjectWithProgressWithContext - creates an object using server-side copying of // existing objects. It takes a list of source objects (with optional // offsets) and concatenates them into a new object using only // server-side copying operations. Optionally takes progress reader hook // for applications to look at current progress. -func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo, progress io.Reader) error { +func (c Client) ComposeObjectWithProgressWithContext(ctx context.Context, dst DestinationInfo, srcs []SourceInfo, progress io.Reader) error { if len(srcs) < 1 || len(srcs) > maxPartsCount { return ErrInvalidArgument("There must be as least one and up to 10000 source objects.") } - ctx := context.Background() srcSizes := make([]int64, len(srcs)) var totalSize, size, totalParts int64 var srcUserMeta map[string]string @@ -478,7 +482,7 @@ func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo // involved, it is being copied wholly and at most 5GiB in // size, emptyfiles are also supported). if (totalParts == 1 && srcs[0].start == -1 && totalSize <= maxPartSize) || (totalSize == 0) { - return c.CopyObjectWithProgress(dst, srcs[0], progress) + return c.CopyObjectWithProgressWithContext(ctx, dst, srcs[0], progress) } // Now, handle multipart-copy cases. diff --git a/api-get-lifecycle.go b/api-get-lifecycle.go index a24d03e37..009963656 100644 --- a/api-get-lifecycle.go +++ b/api-get-lifecycle.go @@ -26,13 +26,18 @@ import ( "github.com/minio/minio-go/v6/pkg/s3utils" ) -// GetBucketLifecycle - get bucket lifecycle. +// GetBucketLifecycle is a wrapper for GetBucketLifecycleWithContext. func (c Client) GetBucketLifecycle(bucketName string) (string, error) { + return c.GetBucketLifecycleWithContext(context.Background(), bucketName) +} + +// GetBucketLifecycleWithContext - get bucket lifecycle. +func (c Client) GetBucketLifecycleWithContext(ctx context.Context, bucketName string) (string, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return "", err } - bucketLifecycle, err := c.getBucketLifecycle(bucketName) + bucketLifecycle, err := c.getBucketLifecycle(ctx, bucketName) if err != nil { errResponse := ToErrorResponse(err) if errResponse.Code == "NoSuchLifecycleConfiguration" { @@ -44,14 +49,14 @@ func (c Client) GetBucketLifecycle(bucketName string) (string, error) { } // Request server for current bucket lifecycle. -func (c Client) getBucketLifecycle(bucketName string) (string, error) { +func (c Client) getBucketLifecycle(ctx context.Context, bucketName string) (string, error) { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("lifecycle", "") // Execute GET on bucket to get lifecycle. - resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, "GET", requestMetadata{ bucketName: bucketName, queryValues: urlValues, }) diff --git a/api-get-policy.go b/api-get-policy.go index bc1d10530..590b66c4b 100644 --- a/api-get-policy.go +++ b/api-get-policy.go @@ -26,13 +26,18 @@ import ( "github.com/minio/minio-go/v6/pkg/s3utils" ) -// GetBucketPolicy - get bucket policy at a given path. +// GetBucketPolicy is a wrapper for GetBucketPolicyWithContext func (c Client) GetBucketPolicy(bucketName string) (string, error) { + return c.GetBucketPolicyWithContext(context.Background(), bucketName) +} + +// GetBucketPolicyWithContext - get bucket policy at a given path. +func (c Client) GetBucketPolicyWithContext(ctx context.Context, bucketName string) (string, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return "", err } - bucketPolicy, err := c.getBucketPolicy(bucketName) + bucketPolicy, err := c.getBucketPolicy(ctx, bucketName) if err != nil { errResponse := ToErrorResponse(err) if errResponse.Code == "NoSuchBucketPolicy" { @@ -44,14 +49,14 @@ func (c Client) GetBucketPolicy(bucketName string) (string, error) { } // Request server for current bucket policy. -func (c Client) getBucketPolicy(bucketName string) (string, error) { +func (c Client) getBucketPolicy(ctx context.Context, bucketName string) (string, error) { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("policy", "") // Execute GET on bucket to list objects. - resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, "GET", requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, diff --git a/api-list.go b/api-list.go index 1bcb64f3d..7823181f3 100644 --- a/api-list.go +++ b/api-list.go @@ -73,7 +73,14 @@ func (c Client) ListBucketsWithContext(ctx context.Context) ([]BucketInfo, error /// Bucket Read Operations. -// ListObjectsV2WithMetadata lists all objects matching the objectPrefix +// ListObjectsV2WithMetadata is the wrapper for ListObjectsV2WithMetadataWithContext +func (c Client) ListObjectsV2WithMetadata(bucketName, objectPrefix string, + recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { + return c.ListObjectsV2WithMetadataWithContext(context.Background(), + bucketName, objectPrefix, recursive, doneCh) +} + +// ListObjectsV2WithMetadataWithContext lists all objects matching the objectPrefix // from the specified bucket. If recursion is enabled it would list // all subdirectories and all its contents. This call adds // UserMetadata information as well for each object. @@ -95,11 +102,12 @@ func (c Client) ListBucketsWithContext(ctx context.Context) ([]BucketInfo, error // recursive := true // // Add metadata // metadata := true -// for message := range api.ListObjectsV2WithMetadata("mytestbucket", "starthere", recursive, doneCh) { +// for message := range api.ListObjectsV2WithMetadataWithContext(ctx, +// "mytestbucket", "starthere", recursive, doneCh) { // fmt.Println(message) // } // -func (c Client) ListObjectsV2WithMetadata(bucketName, objectPrefix string, recursive bool, +func (c Client) ListObjectsV2WithMetadataWithContext(ctx context.Context, bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { // Check whether this is snowball region, if yes ListObjectsV2 doesn't work, fallback to listObjectsV1. if location, ok := c.bucketLocCache.Get(bucketName); ok { @@ -107,10 +115,10 @@ func (c Client) ListObjectsV2WithMetadata(bucketName, objectPrefix string, recur return c.ListObjects(bucketName, objectPrefix, recursive, doneCh) } } - return c.listObjectsV2(bucketName, objectPrefix, recursive, true, doneCh) + return c.listObjectsV2(ctx, bucketName, objectPrefix, recursive, true, doneCh) } -func (c Client) listObjectsV2(bucketName, objectPrefix string, recursive, metadata bool, doneCh <-chan struct{}) <-chan ObjectInfo { +func (c Client) listObjectsV2(ctx context.Context, bucketName, objectPrefix string, recursive, metadata bool, doneCh <-chan struct{}) <-chan ObjectInfo { // Allocate new list objects channel. objectStatCh := make(chan ObjectInfo, 1) // Default listing is delimited at "/" @@ -148,7 +156,7 @@ func (c Client) listObjectsV2(bucketName, objectPrefix string, recursive, metada var continuationToken string for { // Get list of objects a maximum of 1000 per request. - result, err := c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, + result, err := c.listObjectsV2Query(ctx, bucketName, objectPrefix, continuationToken, fetchOwner, metadata, delimiter, 0, "") if err != nil { objectStatCh <- ObjectInfo{ @@ -216,13 +224,18 @@ func (c Client) listObjectsV2(bucketName, objectPrefix string, recursive, metada // } // func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { + return c.ListObjectsV2WithContext(context.Background(), bucketName, objectPrefix, recursive, doneCh) +} + +// ListObjectsV2WithContext does the ListObjectsV2 job with context +func (c Client) ListObjectsV2WithContext(ctx context.Context, bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { // Check whether this is snowball region, if yes ListObjectsV2 doesn't work, fallback to listObjectsV1. if location, ok := c.bucketLocCache.Get(bucketName); ok { if location == "snowball" { - return c.ListObjects(bucketName, objectPrefix, recursive, doneCh) + return c.ListObjectsWithContext(ctx, bucketName, objectPrefix, recursive, doneCh) } } - return c.listObjectsV2(bucketName, objectPrefix, recursive, false, doneCh) + return c.listObjectsV2(ctx, bucketName, objectPrefix, recursive, false, doneCh) } // listObjectsV2Query - (List Objects V2) - List some or all (up to 1000) of the objects in a bucket. @@ -236,7 +249,7 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d // ?max-keys - Sets the maximum number of keys returned in the response body. // ?start-after - Specifies the key to start after when listing objects in a bucket. // ?metadata - Specifies if we want metadata for the objects as part of list operation. -func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken string, fetchOwner, metadata bool, delimiter string, maxkeys int, startAfter string) (ListBucketV2Result, error) { +func (c Client) listObjectsV2Query(ctx context.Context, bucketName, objectPrefix, continuationToken string, fetchOwner, metadata bool, delimiter string, maxkeys int, startAfter string) (ListBucketV2Result, error) { // Validate bucket name. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ListBucketV2Result{}, err @@ -286,7 +299,7 @@ func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken s } // Execute GET on bucket to list objects. - resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, "GET", requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, @@ -334,9 +347,16 @@ func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken s return listBucketResult, nil } -// ListObjects - (List Objects) - List some objects or all recursively. -// -// ListObjects lists all objects matching the objectPrefix from +// ListObjects is a wrapper for ListObjectsWithContext +func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, + doneCh <-chan struct{}) <-chan ObjectInfo { + return c.ListObjectsWithContext(context.Background(), bucketName, + objectPrefix, recursive, doneCh) +} + +// ListObjectsWithContext - (List Objects with context) - List some +// objects or all recursively. +// ListObjectsWithContext lists all objects matching the objectPrefix from // the specified bucket. If recursion is enabled it would list // all subdirectories and all its contents. // @@ -352,11 +372,13 @@ func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken s // defer close(doneCh) // // Recurively list all objects in 'mytestbucket' // recursive := true -// for message := range api.ListObjects("mytestbucket", "starthere", recursive, doneCh) { +// for message := range api.ListObjectsWithContext(ctx, "mytestbucket", +// "starthere", recursive, doneCh) { // fmt.Println(message) // } // -func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { +func (c Client) ListObjectsWithContext(ctx context.Context, bucketName, + objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { // Allocate new list objects channel. objectStatCh := make(chan ObjectInfo, 1) // Default listing is delimited at "/" @@ -389,7 +411,7 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don var marker string for { // Get list of objects a maximum of 1000 per request. - result, err := c.listObjectsQuery(bucketName, objectPrefix, marker, delimiter, 0) + result, err := c.listObjectsQuery(ctx, bucketName, objectPrefix, marker, delimiter, 0) if err != nil { objectStatCh <- ObjectInfo{ Err: err, @@ -445,7 +467,7 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don // ?delimiter - A delimiter is a character you use to group keys. // ?prefix - Limits the response to keys that begin with the specified prefix. // ?max-keys - Sets the maximum number of keys returned in the response body. -func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimiter string, maxkeys int) (ListBucketResult, error) { +func (c Client) listObjectsQuery(ctx context.Context, bucketName, objectPrefix, objectMarker, delimiter string, maxkeys int) (ListBucketResult, error) { // Validate bucket name. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ListBucketResult{}, err @@ -478,7 +500,7 @@ func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimit urlValues.Set("encoding-type", "url") // Execute GET on bucket to list objects. - resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, "GET", requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, @@ -543,15 +565,21 @@ func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimit // for message := range api.ListIncompleteUploads("mytestbucket", "starthere", recursive) { // fmt.Println(message) // } -// + +// ListIncompleteUploads is a wrapper for ListIncompleteUploadsWithContent func (c Client) ListIncompleteUploads(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo { + return c.ListIncompleteUploadsWithContext(context.Background(), bucketName, objectPrefix, recursive, doneCh) +} + +// ListIncompleteUploadsWithContext adds the context argument to ListIncompleteUploads functionality +func (c Client) ListIncompleteUploadsWithContext(ctx context.Context, bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo { // Turn on size aggregation of individual parts. isAggregateSize := true - return c.listIncompleteUploads(bucketName, objectPrefix, recursive, isAggregateSize, doneCh) + return c.listIncompleteUploads(ctx, bucketName, objectPrefix, recursive, isAggregateSize, doneCh) } // listIncompleteUploads lists all incomplete uploads. -func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive, aggregateSize bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo { +func (c Client) listIncompleteUploads(ctx context.Context, bucketName, objectPrefix string, recursive, aggregateSize bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo { // Allocate channel for multipart uploads. objectMultipartStatCh := make(chan ObjectMultipartInfo, 1) // Delimiter is set to "/" by default. @@ -583,7 +611,7 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive var uploadIDMarker string for { // list all multipart uploads. - result, err := c.listMultipartUploadsQuery(bucketName, objectMarker, uploadIDMarker, objectPrefix, delimiter, 0) + result, err := c.listMultipartUploadsQuery(ctx, bucketName, objectMarker, uploadIDMarker, objectPrefix, delimiter, 0) if err != nil { objectMultipartStatCh <- ObjectMultipartInfo{ Err: err, @@ -598,7 +626,7 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive // Calculate total size of the uploaded parts if 'aggregateSize' is enabled. if aggregateSize { // Get total multipart size. - obj.Size, err = c.getTotalMultipartSize(bucketName, obj.Key, obj.UploadID) + obj.Size, err = c.getTotalMultipartSize(ctx, bucketName, obj.Key, obj.UploadID) if err != nil { objectMultipartStatCh <- ObjectMultipartInfo{ Err: err, @@ -633,9 +661,10 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive }(objectMultipartStatCh) // return. return objectMultipartStatCh + } -// listMultipartUploads - (List Multipart Uploads). +// listMultipartUploadsQuery - (List Multipart Uploads). // - Lists some or all (up to 1000) in-progress multipart uploads in a bucket. // // You can use the request parameters as selection criteria to return a subset of the uploads in a bucket. @@ -646,7 +675,7 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive // ?delimiter - A delimiter is a character you use to group keys. // ?prefix - Limits the response to keys that begin with the specified prefix. // ?max-uploads - Sets the maximum number of multipart uploads returned in the response body. -func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker, prefix, delimiter string, maxUploads int) (ListMultipartUploadsResult, error) { +func (c Client) listMultipartUploadsQuery(ctx context.Context, bucketName, keyMarker, uploadIDMarker, prefix, delimiter string, maxUploads int) (ListMultipartUploadsResult, error) { // Get resources properly escaped and lined up before using them in http request. urlValues := make(url.Values) // Set uploads. @@ -676,7 +705,7 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker, } // Execute GET on bucketName to list multipart uploads. - resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, "GET", requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, @@ -725,13 +754,13 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker, } // listObjectParts list all object parts recursively. -func (c Client) listObjectParts(bucketName, objectName, uploadID string) (partsInfo map[int]ObjectPart, err error) { +func (c Client) listObjectParts(ctx context.Context, bucketName, objectName, uploadID string) (partsInfo map[int]ObjectPart, err error) { // Part number marker for the next batch of request. var nextPartNumberMarker int partsInfo = make(map[int]ObjectPart) for { // Get list of uploaded parts a maximum of 1000 per request. - listObjPartsResult, err := c.listObjectPartsQuery(bucketName, objectName, uploadID, nextPartNumberMarker, 1000) + listObjPartsResult, err := c.listObjectPartsQuery(ctx, bucketName, objectName, uploadID, nextPartNumberMarker, 1000) if err != nil { return nil, err } @@ -754,7 +783,7 @@ func (c Client) listObjectParts(bucketName, objectName, uploadID string) (partsI } // findUploadIDs lists all incomplete uploads and find the uploadIDs of the matching object name. -func (c Client) findUploadIDs(bucketName, objectName string) ([]string, error) { +func (c Client) findUploadIDs(ctx context.Context, bucketName, objectName string) ([]string, error) { var uploadIDs []string // Make list incomplete uploads recursive. isRecursive := true @@ -764,7 +793,7 @@ func (c Client) findUploadIDs(bucketName, objectName string) ([]string, error) { doneCh := make(chan struct{}) defer close(doneCh) // List all incomplete uploads. - for mpUpload := range c.listIncompleteUploads(bucketName, objectName, isRecursive, isAggregateSize, doneCh) { + for mpUpload := range c.listIncompleteUploads(ctx, bucketName, objectName, isRecursive, isAggregateSize, doneCh) { if mpUpload.Err != nil { return nil, mpUpload.Err } @@ -776,10 +805,9 @@ func (c Client) findUploadIDs(bucketName, objectName string) ([]string, error) { return uploadIDs, nil } -// getTotalMultipartSize - calculate total uploaded size for the a given multipart object. -func (c Client) getTotalMultipartSize(bucketName, objectName, uploadID string) (size int64, err error) { +func (c Client) getTotalMultipartSize(ctx context.Context, bucketName, objectName, uploadID string) (size int64, err error) { // Iterate over all parts and aggregate the size. - partsInfo, err := c.listObjectParts(bucketName, objectName, uploadID) + partsInfo, err := c.listObjectParts(ctx, bucketName, objectName, uploadID) if err != nil { return 0, err } @@ -799,7 +827,7 @@ func (c Client) getTotalMultipartSize(bucketName, objectName, uploadID string) ( // ?part-number-marker - Specifies the part after which listing should // begin. // ?max-parts - Maximum parts to be listed per request. -func (c Client) listObjectPartsQuery(bucketName, objectName, uploadID string, partNumberMarker, maxParts int) (ListObjectPartsResult, error) { +func (c Client) listObjectPartsQuery(ctx context.Context, bucketName, objectName, uploadID string, partNumberMarker, maxParts int) (ListObjectPartsResult, error) { // Get resources properly escaped and lined up before using them in http request. urlValues := make(url.Values) // Set part number marker. @@ -814,7 +842,7 @@ func (c Client) listObjectPartsQuery(bucketName, objectName, uploadID string, pa } // Execute GET on objectName to get list of parts. - resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, "GET", requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, diff --git a/api-notification.go b/api-notification.go index 24ee121e3..624468e2a 100644 --- a/api-notification.go +++ b/api-notification.go @@ -28,13 +28,18 @@ import ( "github.com/minio/minio-go/v6/pkg/s3utils" ) -// GetBucketNotification - get bucket notification at a given path. +// GetBucketNotification is a wrapper for GetBucketNotificationWithContext func (c Client) GetBucketNotification(bucketName string) (bucketNotification BucketNotification, err error) { + return c.GetBucketNotificationWithContext(context.Background(), bucketName) +} + +// GetBucketNotificationWithContext - get bucket notification at a given path. +func (c Client) GetBucketNotificationWithContext(ctx context.Context, bucketName string) (bucketNotification BucketNotification, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return BucketNotification{}, err } - notification, err := c.getBucketNotification(bucketName) + notification, err := c.getBucketNotification(ctx, bucketName) if err != nil { return BucketNotification{}, err } @@ -42,12 +47,12 @@ func (c Client) GetBucketNotification(bucketName string) (bucketNotification Buc } // Request server for notification rules. -func (c Client) getBucketNotification(bucketName string) (BucketNotification, error) { +func (c Client) getBucketNotification(ctx context.Context, bucketName string) (BucketNotification, error) { urlValues := make(url.Values) urlValues.Set("notification", "") // Execute GET on bucket to list objects. - resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, "GET", requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, @@ -135,8 +140,13 @@ type NotificationInfo struct { Err error } -// ListenBucketNotification - listen on bucket notifications. +// ListenBucketNotification is a wrapper for ListenBucketNotificationWithContext. func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, events []string, doneCh <-chan struct{}) <-chan NotificationInfo { + return c.ListenBucketNotificationWithContext(context.Background(), bucketName, prefix, suffix, events, doneCh) +} + +// ListenBucketNotificationWithContext - listen on bucket notifications. +func (c Client) ListenBucketNotificationWithContext(ctx context.Context, bucketName, prefix, suffix string, events []string, doneCh <-chan struct{}) <-chan NotificationInfo { notificationInfoCh := make(chan NotificationInfo, 1) const notificationCapacity = 1024 * 1024 notificationEventBuffer := make([]byte, notificationCapacity) @@ -182,7 +192,7 @@ func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, even // Wait on the jitter retry loop. for range c.newRetryTimerContinous(time.Second, time.Second*30, MaxJitter, retryDoneCh) { // Execute GET on bucket to list objects. - resp, err := c.executeMethod(context.Background(), http.MethodGet, requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, diff --git a/api-object-legal-hold.go b/api-object-legal-hold.go index 2b43575ac..82665b235 100644 --- a/api-object-legal-hold.go +++ b/api-object-legal-hold.go @@ -80,8 +80,13 @@ func newObjectLegalHold(status *LegalHoldStatus) (*objectLegalHold, error) { return legalHold, nil } -// PutObjectLegalHold : sets object legal hold for a given object and versionID. +// PutObjectLegalHold is the wrapper for PutObjectLegalHoldWithContext. func (c Client) PutObjectLegalHold(bucketName, objectName string, opts PutObjectLegalHoldOptions) error { + return c.PutObjectLegalHoldWithContext(context.Background(), bucketName, objectName, opts) +} + +// PutObjectLegalHoldWithContext : sets object legal hold for a given object and versionID. +func (c Client) PutObjectLegalHoldWithContext(ctx context.Context, bucketName, objectName string, opts PutObjectLegalHoldOptions) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -121,7 +126,7 @@ func (c Client) PutObjectLegalHold(bucketName, objectName string, opts PutObject } // Execute PUT Object Legal Hold. - resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) + resp, err := c.executeMethod(ctx, "PUT", reqMetadata) defer closeResponse(resp) if err != nil { return err diff --git a/api-object-lock.go b/api-object-lock.go index bdd2b07c4..0a40206f9 100644 --- a/api-object-lock.go +++ b/api-object-lock.go @@ -138,8 +138,13 @@ func newObjectLockConfig(mode *RetentionMode, validity *uint, unit *ValidityUnit return nil, fmt.Errorf("all of retention mode, validity and validity unit must be passed") } -// SetBucketObjectLockConfig sets object lock configuration in given bucket. mode, validity and unit are either all set or all nil. +// SetBucketObjectLockConfig is a wrapper for SetBucketObjectLockConfigWithContext. func (c Client) SetBucketObjectLockConfig(bucketName string, mode *RetentionMode, validity *uint, unit *ValidityUnit) error { + return c.SetBucketObjectLockConfigWithContext(context.Background(), bucketName, mode, validity, unit) +} + +// SetBucketObjectLockConfigWithContext sets object lock configuration in given bucket. mode, validity and unit are either all set or all nil. +func (c Client) SetBucketObjectLockConfigWithContext(ctx context.Context, bucketName string, mode *RetentionMode, validity *uint, unit *ValidityUnit) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -170,7 +175,7 @@ func (c Client) SetBucketObjectLockConfig(bucketName string, mode *RetentionMode } // Execute PUT bucket object lock configuration. - resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) + resp, err := c.executeMethod(ctx, "PUT", reqMetadata) defer closeResponse(resp) if err != nil { return err @@ -183,8 +188,13 @@ func (c Client) SetBucketObjectLockConfig(bucketName string, mode *RetentionMode return nil } -// GetObjectLockConfig gets object lock configuration of given bucket. +// GetObjectLockConfig is a wrapper for GetObjectLockConfigWithContext func (c Client) GetObjectLockConfig(bucketName string) (objectLock string, mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { + return c.GetObjectLockConfigWithContext(context.Background(), bucketName) +} + +// GetObjectLockConfigWithContext gets object lock configuration of given bucket. +func (c Client) GetObjectLockConfigWithContext(ctx context.Context, bucketName string) (objectLock string, mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return "", nil, nil, nil, err @@ -194,7 +204,7 @@ func (c Client) GetObjectLockConfig(bucketName string) (objectLock string, mode urlValues.Set("object-lock", "") // Execute GET on bucket to list objects. - resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, "GET", requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, @@ -230,8 +240,14 @@ func (c Client) GetObjectLockConfig(bucketName string) (objectLock string, mode } // GetBucketObjectLockConfig gets object lock configuration of given bucket. -func (c Client) GetBucketObjectLockConfig(bucketName string) (mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { - _, mode, validity, unit, err = c.GetObjectLockConfig(bucketName) +func (c Client) GetBucketObjectLockConfig(bucketName string) ( + mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { + return c.GetBucketObjectLockConfigWithContext(context.Background(), bucketName) +} + +// GetBucketObjectLockConfigWithContext gets object lock configuration of given bucket. +func (c Client) GetBucketObjectLockConfigWithContext(ctx context.Context, bucketName string) (mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { + _, mode, validity, unit, err = c.GetObjectLockConfigWithContext(ctx, bucketName) return mode, validity, unit, err } diff --git a/api-object-retention.go b/api-object-retention.go index 822bcc604..264f1e989 100644 --- a/api-object-retention.go +++ b/api-object-retention.go @@ -65,8 +65,13 @@ type PutObjectRetentionOptions struct { VersionID string } -// PutObjectRetention : sets object retention for a given object and versionID. +// PutObjectRetention is a wrapper for PutObjectRetentionWithContext. func (c Client) PutObjectRetention(bucketName, objectName string, opts PutObjectRetentionOptions) error { + return c.PutObjectRetentionWithContext(context.Background(), bucketName, objectName, opts) +} + +// PutObjectRetentionWithContext : sets object retention for a given object and versionID. +func (c Client) PutObjectRetentionWithContext(ctx context.Context, bucketName, objectName string, opts PutObjectRetentionOptions) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -115,7 +120,7 @@ func (c Client) PutObjectRetention(bucketName, objectName string, opts PutObject } // Execute PUT Object Retention. - resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) + resp, err := c.executeMethod(ctx, "PUT", reqMetadata) defer closeResponse(resp) if err != nil { return err diff --git a/api-presigned.go b/api-presigned.go index 2e27c3f97..ad84e3fa7 100644 --- a/api-presigned.go +++ b/api-presigned.go @@ -18,6 +18,7 @@ package minio import ( + "context" "errors" "net/http" "net/url" @@ -29,7 +30,7 @@ import ( // presignURL - Returns a presigned URL for an input 'method'. // Expires maximum is 7days - ie. 604800 and minimum is 1. -func (c Client) presignURL(method string, bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { +func (c Client) presignURL(ctx context.Context, method string, bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { // Input validation. if method == "" { return nil, ErrInvalidArgument("method cannot be empty.") @@ -60,43 +61,66 @@ func (c Client) presignURL(method string, bucketName string, objectName string, return req.URL, nil } -// PresignedGetObject - Returns a presigned URL to access an object +// PresignedGetObject is a wrapper for PresignedGetObjectWithContext +func (c Client) PresignedGetObject(bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { + return c.PresignedGetObjectWithContext(context.Background(), bucketName, objectName, expires, reqParams) +} + +// PresignedGetObjectWithContext - Returns a presigned URL to access an object // data without credentials. URL can have a maximum expiry of // upto 7days or a minimum of 1sec. Additionally you can override // a set of response headers using the query parameters. -func (c Client) PresignedGetObject(bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { +func (c Client) PresignedGetObjectWithContext(ctx context.Context, bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { if err = s3utils.CheckValidObjectName(objectName); err != nil { return nil, err } - return c.presignURL("GET", bucketName, objectName, expires, reqParams) + return c.presignURL(ctx, "GET", bucketName, objectName, expires, reqParams) } -// PresignedHeadObject - Returns a presigned URL to access object -// metadata without credentials. URL can have a maximum expiry of -// upto 7days or a minimum of 1sec. Additionally you can override -// a set of response headers using the query parameters. +// PresignedHeadObject is the wrapper for PresignedHeadObjectWithContext func (c Client) PresignedHeadObject(bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { if err = s3utils.CheckValidObjectName(objectName); err != nil { return nil, err } - return c.presignURL("HEAD", bucketName, objectName, expires, reqParams) + return c.PresignedHeadObjectWithContext(context.Background(), bucketName, objectName, expires, reqParams) +} + +// PresignedHeadObjectWithContext - Returns a presigned URL to access +// object metadata without credentials. URL can have a maximum expiry +// of upto 7days or a minimum of 1sec. Additionally you can override +// a set of response headers using the query parameters. +func (c Client) PresignedHeadObjectWithContext(ctx context.Context, bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { + if err = s3utils.CheckValidObjectName(objectName); err != nil { + return nil, err + } + return c.presignURL(ctx, "HEAD", bucketName, objectName, expires, reqParams) +} + +// PresignedPutObject is a wrapper for PresignedPutObjectWithContext +func (c Client) PresignedPutObject(bucketName string, objectName string, expires time.Duration) (u *url.URL, err error) { + return c.PresignedPutObjectWithContext(context.Background(), bucketName, objectName, expires) } -// PresignedPutObject - Returns a presigned URL to upload an object +// PresignedPutObjectWithContext - Returns a presigned URL to upload an object // without credentials. URL can have a maximum expiry of upto 7days // or a minimum of 1sec. -func (c Client) PresignedPutObject(bucketName string, objectName string, expires time.Duration) (u *url.URL, err error) { +func (c Client) PresignedPutObjectWithContext(ctx context.Context, bucketName string, objectName string, expires time.Duration) (u *url.URL, err error) { if err = s3utils.CheckValidObjectName(objectName); err != nil { return nil, err } - return c.presignURL("PUT", bucketName, objectName, expires, nil) + return c.presignURL(ctx, "PUT", bucketName, objectName, expires, nil) } -// Presign - returns a presigned URL for any http method of your choice +// Presign is a wrapper for PresignWithContext +func (c Client) Presign(method string, bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { + return c.presignURL(context.Background(), method, bucketName, objectName, expires, reqParams) +} + +// PresignWithContext - returns a presigned URL for any http method of your choice // along with custom request params. URL can have a maximum expiry of // upto 7days or a minimum of 1sec. -func (c Client) Presign(method string, bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { - return c.presignURL(method, bucketName, objectName, expires, reqParams) +func (c Client) PresignWithContext(ctx context.Context, method string, bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { + return c.presignURL(ctx, method, bucketName, objectName, expires, reqParams) } // PresignedPostPolicy - Returns POST urlString, form data to upload an object. diff --git a/api-put-bucket.go b/api-put-bucket.go index 9d2486ad0..959d6ed21 100644 --- a/api-put-bucket.go +++ b/api-put-bucket.go @@ -171,12 +171,12 @@ func (c Client) MakeBucketWithObjectLockWithContext(ctx context.Context, bucketN return c.makeBucket(ctx, bucketName, location, true) } -// SetBucketPolicy set the access permissions on an existing bucket. +// SetBucketPolicy sets the access permissions on an existing bucket. func (c Client) SetBucketPolicy(bucketName, policy string) error { return c.SetBucketPolicyWithContext(context.Background(), bucketName, policy) } -// SetBucketPolicyWithContext set the access permissions on an existing bucket. +// SetBucketPolicyWithContext sets the access permissions on an existing bucket. func (c Client) SetBucketPolicyWithContext(ctx context.Context, bucketName, policy string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { @@ -465,9 +465,14 @@ func (c Client) SetBucketNotificationWithContext(ctx context.Context, bucketName return nil } -// RemoveAllBucketNotification - Remove bucket notification clears all previously specified config +// RemoveAllBucketNotification is a wrapper for RemoveAllBucketNotificationWithContext func (c Client) RemoveAllBucketNotification(bucketName string) error { - return c.SetBucketNotification(bucketName, BucketNotification{}) + return c.RemoveAllBucketNotificationWithContext(context.Background(), bucketName) +} + +// RemoveAllBucketNotificationWithContext - Remove bucket notification clears all previously specified config +func (c Client) RemoveAllBucketNotificationWithContext(ctx context.Context, bucketName string) error { + return c.SetBucketNotificationWithContext(ctx, bucketName, BucketNotification{}) } var ( diff --git a/api-put-object-copy.go b/api-put-object-copy.go index aa5a36c4d..d2f8b7f4b 100644 --- a/api-put-object-copy.go +++ b/api-put-object-copy.go @@ -33,9 +33,15 @@ func (c Client) CopyObject(dst DestinationInfo, src SourceInfo) error { return c.CopyObjectWithProgress(dst, src, nil) } -// CopyObjectWithProgress - copy a source object into a new object, optionally takes +// CopyObjectWithProgress is a wrapper for CopyObjectWithProgressWithContext // progress bar input to notify current progress. func (c Client) CopyObjectWithProgress(dst DestinationInfo, src SourceInfo, progress io.Reader) error { + return c.CopyObjectWithProgressWithContext(context.Background(), dst, src, progress) +} + +// CopyObjectWithProgressWithContext - copy a source object into a new object, optionally takes +// progress bar input to notify current progress. +func (c Client) CopyObjectWithProgressWithContext(ctx context.Context, dst DestinationInfo, src SourceInfo, progress io.Reader) error { header := make(http.Header) for k, v := range src.Headers { header[k] = v @@ -76,7 +82,7 @@ func (c Client) CopyObjectWithProgress(dst DestinationInfo, src SourceInfo, prog header.Set(k, v) } - resp, err := c.executeMethod(context.Background(), "PUT", requestMetadata{ + resp, err := c.executeMethod(ctx, "PUT", requestMetadata{ bucketName: dst.bucket, objectName: dst.object, customHeader: header, diff --git a/api-remove.go b/api-remove.go index f2879e8b5..ea916d742 100644 --- a/api-remove.go +++ b/api-remove.go @@ -28,17 +28,22 @@ import ( "github.com/minio/minio-go/v6/pkg/s3utils" ) -// RemoveBucket deletes the bucket name. +// RemoveBucket is the wrapper for RemoveBucketWithContext +func (c Client) RemoveBucket(bucketName string) error { + return c.RemoveBucketWithContext(context.Background(), bucketName) +} + +// RemoveBucketWithContext deletes the bucket name. // // All objects (including all object versions and delete markers). // in the bucket must be deleted before successfully attempting this request. -func (c Client) RemoveBucket(bucketName string) error { +func (c Client) RemoveBucketWithContext(ctx context.Context, bucketName string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err } // Execute DELETE on bucket. - resp, err := c.executeMethod(context.Background(), "DELETE", requestMetadata{ + resp, err := c.executeMethod(ctx, "DELETE", requestMetadata{ bucketName: bucketName, contentSHA256Hex: emptySHA256Hex, }) @@ -183,32 +188,6 @@ func (c Client) RemoveObjectsWithContext(ctx context.Context, bucketName string, return errorCh } -// RemoveObjectsWithOptionsContext - Identical to RemoveObjects call, but accepts context to -// facilitate request cancellation and options to bypass governance retention -func (c Client) RemoveObjectsWithOptionsContext(ctx context.Context, bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError { - errorCh := make(chan RemoveObjectError, 1) - - // Validate if bucket name is valid. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - defer close(errorCh) - errorCh <- RemoveObjectError{ - Err: err, - } - return errorCh - } - // Validate objects channel to be properly allocated. - if objectsCh == nil { - defer close(errorCh) - errorCh <- RemoveObjectError{ - Err: ErrInvalidArgument("Objects channel cannot be nil"), - } - return errorCh - } - - go c.removeObjects(ctx, bucketName, objectsCh, errorCh, opts) - return errorCh -} - // Generate and call MultiDelete S3 requests based on entries received from objectsCh func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh <-chan string, errorCh chan<- RemoveObjectError, opts RemoveObjectsOptions) { maxEntries := 1000 @@ -298,11 +277,42 @@ type RemoveObjectsOptions struct { // The list of objects to remove are received from objectsCh. // Remove failures are sent back via error channel. func (c Client) RemoveObjectsWithOptions(bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError { - return c.RemoveObjectsWithOptionsContext(context.Background(), bucketName, objectsCh, opts) + return c.RemoveObjectsWithOptionsWithContext(context.Background(), bucketName, objectsCh, opts) +} + +// RemoveObjectsWithOptionsWithContext - Identical to RemoveObjects call, but accepts context to +// facilitate request cancellation and options to bypass governance retention +func (c Client) RemoveObjectsWithOptionsWithContext(ctx context.Context, bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError { + errorCh := make(chan RemoveObjectError, 1) + + // Validate if bucket name is valid. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + defer close(errorCh) + errorCh <- RemoveObjectError{ + Err: err, + } + return errorCh + } + // Validate objects channel to be properly allocated. + if objectsCh == nil { + defer close(errorCh) + errorCh <- RemoveObjectError{ + Err: ErrInvalidArgument("Objects channel cannot be nil"), + } + return errorCh + } + + go c.removeObjects(ctx, bucketName, objectsCh, errorCh, opts) + return errorCh } -// RemoveIncompleteUpload aborts an partially uploaded object. +// RemoveIncompleteUpload is a wrapper for RemoveIncompleteUploadWithContext func (c Client) RemoveIncompleteUpload(bucketName, objectName string) error { + return c.RemoveIncompleteUploadWithContext(context.Background(), bucketName, objectName) +} + +// RemoveIncompleteUploadWithContext aborts an partially uploaded object. +func (c Client) RemoveIncompleteUploadWithContext(ctx context.Context, bucketName, objectName string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -311,14 +321,14 @@ func (c Client) RemoveIncompleteUpload(bucketName, objectName string) error { return err } // Find multipart upload ids of the object to be aborted. - uploadIDs, err := c.findUploadIDs(bucketName, objectName) + uploadIDs, err := c.findUploadIDs(ctx, bucketName, objectName) if err != nil { return err } for _, uploadID := range uploadIDs { // abort incomplete multipart upload, based on the upload id passed. - err := c.abortMultipartUpload(context.Background(), bucketName, objectName, uploadID) + err := c.abortMultipartUpload(ctx, bucketName, objectName, uploadID) if err != nil { return err } diff --git a/core.go b/core.go index 6d1fddb2e..b54b8d379 100644 --- a/core.go +++ b/core.go @@ -46,13 +46,13 @@ func NewCore(endpoint string, accessKeyID, secretAccessKey string, secure bool) // ListObjects - List all the objects at a prefix, optionally with marker and delimiter // you can further filter the results. func (c Core) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListBucketResult, err error) { - return c.listObjectsQuery(bucket, prefix, marker, delimiter, maxKeys) + return c.listObjectsQuery(context.Background(), bucket, prefix, marker, delimiter, maxKeys) } // ListObjectsV2 - Lists all the objects at a prefix, similar to ListObjects() but uses // continuationToken instead of marker to support iteration over the results. func (c Core) ListObjectsV2(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int, startAfter string) (ListBucketV2Result, error) { - return c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, false, delimiter, maxkeys, startAfter) + return c.listObjectsV2Query(context.Background(), bucketName, objectPrefix, continuationToken, fetchOwner, false, delimiter, maxkeys, startAfter) } // CopyObjectWithContext - copies an object from source object to destination object on server side. @@ -101,7 +101,7 @@ func (c Core) NewMultipartUpload(bucket, object string, opts PutObjectOptions) ( // ListMultipartUploads - List incomplete uploads. func (c Core) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartUploadsResult, err error) { - return c.listMultipartUploadsQuery(bucket, keyMarker, uploadIDMarker, prefix, delimiter, maxUploads) + return c.listMultipartUploadsQuery(context.Background(), bucket, keyMarker, uploadIDMarker, prefix, delimiter, maxUploads) } // PutObjectPartWithContext - Upload an object part. @@ -115,8 +115,8 @@ func (c Core) PutObjectPart(bucket, object, uploadID string, partID int, data io } // ListObjectParts - List uploaded parts of an incomplete upload.x -func (c Core) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListObjectPartsResult, err error) { - return c.listObjectPartsQuery(bucket, object, uploadID, partNumberMarker, maxParts) +func (c Core) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListObjectPartsResult, err error) { + return c.listObjectPartsQuery(ctx, bucket, object, uploadID, partNumberMarker, maxParts) } // CompleteMultipartUploadWithContext - Concatenate uploaded parts and commit to an object. @@ -144,7 +144,7 @@ func (c Core) AbortMultipartUpload(bucket, object, uploadID string) error { // GetBucketPolicy - fetches bucket access policy for a given bucket. func (c Core) GetBucketPolicy(bucket string) (string, error) { - return c.getBucketPolicy(bucket) + return c.getBucketPolicy(context.Background(), bucket) } // PutBucketPolicy - applies a new bucket access policy for a given bucket. From a57fec8037ec86c0f84116a7ccec82666b71dcd5 Mon Sep 17 00:00:00 2001 From: ebozduman Date: Thu, 11 Jun 2020 17:16:54 -0700 Subject: [PATCH 164/215] Revert breaking name change of RemoveObjectsWithOptionsContext (#1304) --- api-remove.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api-remove.go b/api-remove.go index ea916d742..4160b0d3a 100644 --- a/api-remove.go +++ b/api-remove.go @@ -277,12 +277,12 @@ type RemoveObjectsOptions struct { // The list of objects to remove are received from objectsCh. // Remove failures are sent back via error channel. func (c Client) RemoveObjectsWithOptions(bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError { - return c.RemoveObjectsWithOptionsWithContext(context.Background(), bucketName, objectsCh, opts) + return c.RemoveObjectsWithOptionsContext(context.Background(), bucketName, objectsCh, opts) } -// RemoveObjectsWithOptionsWithContext - Identical to RemoveObjects call, but accepts context to +// RemoveObjectsWithOptionsContext - Identical to RemoveObjects call, but accepts context to // facilitate request cancellation and options to bypass governance retention -func (c Client) RemoveObjectsWithOptionsWithContext(ctx context.Context, bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError { +func (c Client) RemoveObjectsWithOptionsContext(ctx context.Context, bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError { errorCh := make(chan RemoveObjectError, 1) // Validate if bucket name is valid. From e5cd2d129325874148be41ea392e7dfde30ff584 Mon Sep 17 00:00:00 2001 From: Anis Elleuch Date: Tue, 23 Jun 2020 22:30:17 +0100 Subject: [PATCH 165/215] api refactor Remove WithContext APIs move to v7 (#1307) --- README.md | 12 +- README_zh_CN.md | 24 +- api-bucket-tagging.go | 34 +- api-compose-object.go | 27 +- api-error-response.go | 2 +- api-get-bucket-encryption.go | 12 +- api-get-bucket-versioning.go | 14 +- api-get-lifecycle.go | 11 +- api-get-object-acl-context.go | 140 ---- api-get-object-acl.go | 119 ++- api-get-object-context.go | 26 - api-get-object-file.go | 18 +- api-get-object.go | 9 +- api-get-options.go | 2 +- api-get-policy.go | 11 +- api-list.go | 106 +-- api-notification.go | 20 +- api-object-legal-hold.go | 15 +- api-object-lock.go | 34 +- api-object-retention.go | 15 +- api-object-tagging.go | 31 +- api-presigned.go | 49 +- api-put-bucket.go | 95 +-- api-put-object-common.go | 2 +- api-put-object-context.go | 33 - api-put-object-copy.go | 19 +- api-put-object-file-context.go | 8 +- api-put-object-file.go | 27 - api-put-object-multipart.go | 4 +- api-put-object-streaming.go | 2 +- api-put-object.go | 13 +- api-remove.go | 51 +- api-select.go | 4 +- api-stat.go | 19 +- api.go | 14 +- api_unit_test.go | 4 +- bucket-cache.go | 13 +- bucket-cache_test.go | 4 +- bucket-notification.go | 2 +- core.go | 101 +-- core_test.go | 105 +-- docs/API.md | 642 ++++---------- docs/zh_CN/API.md | 20 +- examples/minio/listenbucketnotification.go | 5 +- examples/s3/bucketexists.go | 5 +- examples/s3/composeobject.go | 7 +- examples/s3/copyobject-with-new-tags.go | 5 +- examples/s3/copyobject.go | 5 +- examples/s3/deletebucketencryption.go | 5 +- examples/s3/disableversioning.go | 5 +- examples/s3/enableversioning.go | 5 +- examples/s3/fgetobject-context.go | 54 -- examples/s3/fgetobject.go | 5 +- examples/s3/fputencrypted-object.go | 7 +- examples/s3/fputobject-context.go | 53 -- examples/s3/fputobject.go | 5 +- examples/s3/get-encrypted-object.go | 7 +- examples/s3/getbucketencryption.go | 5 +- examples/s3/getbucketlifecycle.go | 5 +- examples/s3/getbucketnotification.go | 5 +- examples/s3/getbucketpolicy.go | 5 +- examples/s3/getbuckettagging.go | 5 +- examples/s3/getbucketversioning.go | 5 +- examples/s3/getobject-client-encryption.go | 5 +- examples/s3/getobject-context.go | 73 -- examples/s3/getobject.go | 5 +- examples/s3/getobjectacl.go | 5 +- examples/s3/getobjectlegalhold.go | 5 +- examples/s3/getobjectlockconfig.go | 6 +- examples/s3/getobjectretention.go | 7 +- examples/s3/getobjecttagging.go | 5 +- examples/s3/listbuckets.go | 5 +- examples/s3/listincompleteuploads.go | 5 +- examples/s3/listobjects-N.go | 5 +- examples/s3/listobjects.go | 5 +- examples/s3/listobjectsV2.go | 5 +- examples/s3/listobjectsV2WithMetadata.go | 5 +- examples/s3/makebucket.go | 5 +- examples/s3/makebucketwithobjectlock.go | 5 +- examples/s3/presignedgetobject.go | 5 +- examples/s3/presignedheadobject.go | 5 +- examples/s3/presignedpostpolicy.go | 5 +- examples/s3/presignedputobject.go | 5 +- examples/s3/put-encrypted-object.go | 7 +- examples/s3/putobject-client-encryption.go | 5 +- examples/s3/putobject-context.go | 68 -- examples/s3/putobject-getobject-sse.go | 9 +- examples/s3/putobject-progress.go | 7 +- examples/s3/putobject-s3-accelerate.go | 5 +- examples/s3/putobject-streaming.go | 5 +- examples/s3/putobject-with-tags.go | 5 +- examples/s3/putobject.go | 5 +- examples/s3/putobjectlegalhold.go | 5 +- examples/s3/putobjectretention.go | 5 +- examples/s3/putobjecttagging.go | 5 +- examples/s3/removeallbucketnotification.go | 5 +- examples/s3/removebucket.go | 5 +- examples/s3/removebuckettagging.go | 5 +- examples/s3/removeincompleteupload.go | 5 +- examples/s3/removeobject.go | 5 +- examples/s3/removeobjectoptions.go | 5 +- examples/s3/removeobjects.go | 7 +- examples/s3/removeobjecttagging.go | 5 +- examples/s3/selectobject.go | 2 +- examples/s3/setbucketencryption.go | 5 +- examples/s3/setbucketlifecycle.go | 5 +- examples/s3/setbucketnotification.go | 5 +- examples/s3/setbucketpolicy.go | 5 +- examples/s3/setbuckettagging.go | 7 +- examples/s3/setobjectlockconfig.go | 5 +- examples/s3/statobject.go | 5 +- functional_tests.go | 926 ++++++++++----------- go.mod | 8 +- go.sum | 13 +- pkg/credentials/assume_role.go | 2 +- pkg/policy/bucket-policy-condition.go | 2 +- pkg/policy/bucket-policy-condition_test.go | 2 +- pkg/policy/bucket-policy.go | 2 +- pkg/policy/bucket-policy_test.go | 2 +- pkg/signer/request-signature-v2.go | 2 +- pkg/signer/request-signature-v4.go | 2 +- utils.go | 2 +- utils_test.go | 2 +- 123 files changed, 1240 insertions(+), 2224 deletions(-) delete mode 100644 api-get-object-acl-context.go delete mode 100644 api-get-object-context.go delete mode 100644 api-put-object-context.go delete mode 100644 api-put-object-file.go delete mode 100644 examples/s3/fgetobject-context.go delete mode 100644 examples/s3/fputobject-context.go delete mode 100644 examples/s3/getobject-context.go delete mode 100644 examples/s3/putobject-context.go diff --git a/README.md b/README.md index 33eb6c44d..d0b82c1cc 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This document assumes that you have a working [Go development environment](https ## Download from Github ```sh -GO111MODULE=on go get github.com/minio/minio-go/v6 +GO111MODULE=on go get github.com/minio/minio-go/v7 ``` ## Initialize MinIO Client @@ -17,7 +17,7 @@ MinIO client requires the following four parameters specified to connect to an A | Parameter | Description| | :--- | :--- | | endpoint | URL to object storage service. | -| accessKeyID | Access key is the user ID that uniquely identifies your account. | +| accessKeyID | Access key is the user ID that uniquely identifies your account. | | secretAccessKey | Secret key is the password to your account. | | secure | Set this value to 'true' to enable secure (HTTPS) access. | @@ -26,7 +26,7 @@ MinIO client requires the following four parameters specified to connect to an A package main import ( - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" "log" ) @@ -56,7 +56,7 @@ We will use the MinIO server running at [https://play.min.io](https://play.min.i package main import ( - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" "log" ) @@ -141,14 +141,10 @@ The full API Reference is available here. ### API Reference : File Object Operations * [`FPutObject`](https://docs.min.io/docs/golang-client-api-reference#FPutObject) * [`FGetObject`](https://docs.min.io/docs/golang-client-api-reference#FGetObject) -* [`FPutObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#FPutObjectWithContext) -* [`FGetObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#FGetObjectWithContext) ### API Reference : Object Operations * [`GetObject`](https://docs.min.io/docs/golang-client-api-reference#GetObject) * [`PutObject`](https://docs.min.io/docs/golang-client-api-reference#PutObject) -* [`GetObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#GetObjectWithContext) -* [`PutObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#PutObjectWithContext) * [`PutObjectStreaming`](https://docs.min.io/docs/golang-client-api-reference#PutObjectStreaming) * [`StatObject`](https://docs.min.io/docs/golang-client-api-reference#StatObject) * [`CopyObject`](https://docs.min.io/docs/golang-client-api-reference#CopyObject) diff --git a/README_zh_CN.md b/README_zh_CN.md index b5ba0c1d0..0911b0905 100644 --- a/README_zh_CN.md +++ b/README_zh_CN.md @@ -2,7 +2,7 @@ MinIO Go Client SDK提供了简单的API来访问任何与Amazon S3兼容的对象存储服务。 -**支持的云存储:** +**支持的云存储:** - AWS Signature Version 4 - Amazon S3 @@ -26,10 +26,10 @@ go get -u github.com/minio/minio-go ## 初始化MinIO Client MinIO client需要以下4个参数来连接与Amazon S3兼容的对象存储。 -| 参数 | 描述| +| 参数 | 描述| | :--- | :--- | -| endpoint | 对象存储服务的URL | -| accessKeyID | Access key是唯一标识你的账户的用户ID。 | +| endpoint | 对象存储服务的URL | +| accessKeyID | Access key是唯一标识你的账户的用户ID。 | | secretAccessKey | Secret key是你账户的密码。 | | secure | true代表使用HTTPS | @@ -38,7 +38,7 @@ MinIO client需要以下4个参数来连接与Amazon S3兼容的对象存储。 package main import ( - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" "log" ) @@ -68,7 +68,7 @@ func main() { package main import ( - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" "log" ) @@ -118,7 +118,7 @@ func main() { ### 运行FileUploader ```sh go run file-uploader.go -2016/08/13 17:03:28 Successfully created mymusic +2016/08/13 17:03:28 Successfully created mymusic 2016/08/13 17:03:40 Successfully uploaded golden-oldies.zip of size 16253413 mc ls play/mymusic/ @@ -151,14 +151,10 @@ mc ls play/mymusic/ ### API文档 : 操作文件对象 * [`FPutObject`](https://docs.min.io/docs/golang-client-api-reference#FPutObject) * [`FGetObject`](https://docs.min.io/docs/golang-client-api-reference#FPutObject) -* [`FPutObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#FPutObjectWithContext) -* [`FGetObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#FGetObjectWithContext) ### API文档 : 操作对象 * [`GetObject`](https://docs.min.io/docs/golang-client-api-reference#GetObject) * [`PutObject`](https://docs.min.io/docs/golang-client-api-reference#PutObject) -* [`GetObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#GetObjectWithContext) -* [`PutObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#PutObjectWithContext) * [`PutObjectStreaming`](https://docs.min.io/docs/golang-client-api-reference#PutObjectStreaming) * [`StatObject`](https://docs.min.io/docs/golang-client-api-reference#StatObject) * [`CopyObject`](https://docs.min.io/docs/golang-client-api-reference#CopyObject) @@ -197,7 +193,7 @@ mc ls play/mymusic/ * [setbucketpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketpolicy.go) * [getbucketpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketpolicy.go) * [listbucketpolicies.go](https://github.com/minio/minio-go/blob/master/examples/s3/listbucketpolicies.go) - + ### 完整示例 : 存储桶通知 * [setbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketnotification.go) * [getbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketnotification.go) @@ -238,7 +234,3 @@ mc ls play/mymusic/ ## 贡献 [贡献指南](https://github.com/minio/minio-go/blob/master/docs/zh_CN/CONTRIBUTING.md) - -[![Build Status](https://travis-ci.org/minio/minio-go.svg)](https://travis-ci.org/minio/minio-go) -[![Build status](https://ci.appveyor.com/api/projects/status/1d05e6nvxcelmrak?svg=true)](https://ci.appveyor.com/project/harshavardhana/minio-go) - diff --git a/api-bucket-tagging.go b/api-bucket-tagging.go index ab1828b77..ded968b94 100644 --- a/api-bucket-tagging.go +++ b/api-bucket-tagging.go @@ -26,17 +26,13 @@ import ( "net/http" "net/url" - "github.com/minio/minio-go/v6/pkg/s3utils" - "github.com/minio/minio-go/v6/pkg/tags" + "github.com/minio/minio-go/v7/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/tags" ) -// GetBucketTagging gets tagging configuration for a bucket. -func (c Client) GetBucketTagging(bucketName string) (*tags.Tags, error) { - return c.GetBucketTaggingWithContext(context.Background(), bucketName) -} - -// GetBucketTaggingWithContext gets tagging configuration for a bucket with a context to control cancellations and timeouts. -func (c Client) GetBucketTaggingWithContext(ctx context.Context, bucketName string) (*tags.Tags, error) { +// GetBucketTagging fetch tagging configuration for a bucket with a +// context to control cancellations and timeouts. +func (c Client) GetBucketTagging(ctx context.Context, bucketName string) (*tags.Tags, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return nil, err @@ -66,13 +62,9 @@ func (c Client) GetBucketTaggingWithContext(ctx context.Context, bucketName stri return tags.ParseBucketXML(resp.Body) } -// SetBucketTagging sets tagging configuration for a bucket. -func (c Client) SetBucketTagging(bucketName string, tags *tags.Tags) error { - return c.SetBucketTaggingWithContext(context.Background(), bucketName, tags) -} - -// SetBucketTaggingWithContext sets tagging configuration for a bucket with a context to control cancellations and timeouts. -func (c Client) SetBucketTaggingWithContext(ctx context.Context, bucketName string, tags *tags.Tags) error { +// SetBucketTagging sets tagging configuration for a bucket +// with a context to control cancellations and timeouts. +func (c Client) SetBucketTagging(ctx context.Context, bucketName string, tags *tags.Tags) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -113,13 +105,9 @@ func (c Client) SetBucketTaggingWithContext(ctx context.Context, bucketName stri return nil } -// DeleteBucketTagging removes tagging configuration for a bucket. -func (c Client) DeleteBucketTagging(bucketName string) error { - return c.DeleteBucketTaggingWithContext(context.Background(), bucketName) -} - -// DeleteBucketTaggingWithContext removes tagging configuration for a bucket with a context to control cancellations and timeouts. -func (c Client) DeleteBucketTaggingWithContext(ctx context.Context, bucketName string) error { +// DeleteBucketTagging removes tagging configuration for a +// bucket with a context to control cancellations and timeouts. +func (c Client) DeleteBucketTagging(ctx context.Context, bucketName string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err diff --git a/api-compose-object.go b/api-compose-object.go index ef1d24440..10df794eb 100644 --- a/api-compose-object.go +++ b/api-compose-object.go @@ -28,8 +28,8 @@ import ( "strings" "time" - "github.com/minio/minio-go/v6/pkg/encrypt" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/encrypt" + "github.com/minio/minio-go/v7/pkg/s3utils" ) // DestinationInfo - type with information about the object to be @@ -408,17 +408,12 @@ func (c Client) uploadPartCopy(ctx context.Context, bucket, object, uploadID str return p, nil } -// ComposeObjectWithProgress is a wrapper for ComposeObjectWithProgressWithContext. -func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo, progress io.Reader) error { - return c.ComposeObjectWithProgressWithContext(context.Background(), dst, srcs, progress) -} - -// ComposeObjectWithProgressWithContext - creates an object using server-side copying of -// existing objects. It takes a list of source objects (with optional -// offsets) and concatenates them into a new object using only -// server-side copying operations. Optionally takes progress reader hook -// for applications to look at current progress. -func (c Client) ComposeObjectWithProgressWithContext(ctx context.Context, dst DestinationInfo, srcs []SourceInfo, progress io.Reader) error { +// ComposeObjectWithProgress - creates an object using server-side copying +// of existing objects. It takes a list of source objects (with optional offsets) +// and concatenates them into a new object using only server-side copying +// operations. Optionally takes progress reader hook for applications to +// look at current progress. +func (c Client) ComposeObjectWithProgress(ctx context.Context, dst DestinationInfo, srcs []SourceInfo, progress io.Reader) error { if len(srcs) < 1 || len(srcs) > maxPartsCount { return ErrInvalidArgument("There must be as least one and up to 10000 source objects.") } @@ -482,7 +477,7 @@ func (c Client) ComposeObjectWithProgressWithContext(ctx context.Context, dst De // involved, it is being copied wholly and at most 5GiB in // size, emptyfiles are also supported). if (totalParts == 1 && srcs[0].start == -1 && totalSize <= maxPartSize) || (totalSize == 0) { - return c.CopyObjectWithProgressWithContext(ctx, dst, srcs[0], progress) + return c.CopyObjectWithProgress(ctx, dst, srcs[0], progress) } // Now, handle multipart-copy cases. @@ -566,8 +561,8 @@ func (c Client) ComposeObjectWithProgressWithContext(ctx context.Context, dst De // existing objects. It takes a list of source objects (with optional // offsets) and concatenates them into a new object using only // server-side copying operations. -func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error { - return c.ComposeObjectWithProgress(dst, srcs, nil) +func (c Client) ComposeObject(ctx context.Context, dst DestinationInfo, srcs []SourceInfo) error { + return c.ComposeObjectWithProgress(ctx, dst, srcs, nil) } // partsRequired is maximum parts possible with diff --git a/api-error-response.go b/api-error-response.go index 4fda64856..4628f2c3e 100644 --- a/api-error-response.go +++ b/api-error-response.go @@ -63,7 +63,7 @@ type ErrorResponse struct { // // For example: // -// import s3 "github.com/minio/minio-go/v6" +// import s3 "github.com/minio/minio-go/v7" // ... // ... // reader, stat, err := s3.GetObject(...) diff --git a/api-get-bucket-encryption.go b/api-get-bucket-encryption.go index 213abb8fc..3f85668d0 100644 --- a/api-get-bucket-encryption.go +++ b/api-get-bucket-encryption.go @@ -23,16 +23,12 @@ import ( "net/http" "net/url" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) -// GetBucketEncryption - get default encryption configuration for a bucket. -func (c Client) GetBucketEncryption(bucketName string) (ServerSideEncryptionConfiguration, error) { - return c.GetBucketEncryptionWithContext(context.Background(), bucketName) -} - -// GetBucketEncryptionWithContext gets the default encryption configuration on an existing bucket with a context to control cancellations and timeouts. -func (c Client) GetBucketEncryptionWithContext(ctx context.Context, bucketName string) (ServerSideEncryptionConfiguration, error) { +// GetBucketEncryption gets the default encryption configuration +// on an existing bucket with a context to control cancellations and timeouts. +func (c Client) GetBucketEncryption(ctx context.Context, bucketName string) (ServerSideEncryptionConfiguration, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ServerSideEncryptionConfiguration{}, err diff --git a/api-get-bucket-versioning.go b/api-get-bucket-versioning.go index 9b13d6730..01458e85d 100644 --- a/api-get-bucket-versioning.go +++ b/api-get-bucket-versioning.go @@ -23,23 +23,19 @@ import ( "net/http" "net/url" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) // BucketVersioningConfiguration is the versioning configuration structure type BucketVersioningConfiguration struct { XMLName xml.Name `xml:"VersioningConfiguration"` Status string `xml:"Status"` - MfaDelete string `xml:"MfaDelete,omitempty"` + MFADelete string `xml:"MfaDelete,omitempty"` } -// GetBucketVersioning - get versioning configuration for a bucket. -func (c Client) GetBucketVersioning(bucketName string) (BucketVersioningConfiguration, error) { - return c.GetBucketVersioningWithContext(context.Background(), bucketName) -} - -// GetBucketVersioningWithContext gets the versioning configuration on an existing bucket with a context to control cancellations and timeouts. -func (c Client) GetBucketVersioningWithContext(ctx context.Context, bucketName string) (BucketVersioningConfiguration, error) { +// GetBucketVersioning gets the versioning configuration on +// an existing bucket with a context to control cancellations and timeouts. +func (c Client) GetBucketVersioning(ctx context.Context, bucketName string) (BucketVersioningConfiguration, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return BucketVersioningConfiguration{}, err diff --git a/api-get-lifecycle.go b/api-get-lifecycle.go index 009963656..ff349f24a 100644 --- a/api-get-lifecycle.go +++ b/api-get-lifecycle.go @@ -23,16 +23,11 @@ import ( "net/http" "net/url" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) -// GetBucketLifecycle is a wrapper for GetBucketLifecycleWithContext. -func (c Client) GetBucketLifecycle(bucketName string) (string, error) { - return c.GetBucketLifecycleWithContext(context.Background(), bucketName) -} - -// GetBucketLifecycleWithContext - get bucket lifecycle. -func (c Client) GetBucketLifecycleWithContext(ctx context.Context, bucketName string) (string, error) { +// GetBucketLifecycle fetch bucket lifecycle configuration +func (c Client) GetBucketLifecycle(ctx context.Context, bucketName string) (string, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return "", err diff --git a/api-get-object-acl-context.go b/api-get-object-acl-context.go deleted file mode 100644 index f49984f68..000000000 --- a/api-get-object-acl-context.go +++ /dev/null @@ -1,140 +0,0 @@ -/* - * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2018 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "context" - "net/http" - "net/url" -) - -type accessControlPolicy struct { - Owner struct { - ID string `xml:"ID"` - DisplayName string `xml:"DisplayName"` - } `xml:"Owner"` - AccessControlList struct { - Grant []struct { - Grantee struct { - ID string `xml:"ID"` - DisplayName string `xml:"DisplayName"` - URI string `xml:"URI"` - } `xml:"Grantee"` - Permission string `xml:"Permission"` - } `xml:"Grant"` - } `xml:"AccessControlList"` -} - -//GetObjectACLWithContext get object ACLs -func (c Client) GetObjectACLWithContext(ctx context.Context, bucketName, objectName string) (*ObjectInfo, error) { - resp, err := c.executeMethod(ctx, "GET", requestMetadata{ - bucketName: bucketName, - objectName: objectName, - queryValues: url.Values{ - "acl": []string{""}, - }, - }) - if err != nil { - return nil, err - } - defer closeResponse(resp) - - if resp.StatusCode != http.StatusOK { - return nil, httpRespToErrorResponse(resp, bucketName, objectName) - } - - res := &accessControlPolicy{} - - if err := xmlDecoder(resp.Body, res); err != nil { - return nil, err - } - - objInfo, err := c.statObject(ctx, bucketName, objectName, StatObjectOptions{}) - if err != nil { - return nil, err - } - - objInfo.Owner.DisplayName = res.Owner.DisplayName - objInfo.Owner.ID = res.Owner.ID - - objInfo.Grant = append(objInfo.Grant, res.AccessControlList.Grant...) - - cannedACL := getCannedACL(res) - if cannedACL != "" { - objInfo.Metadata.Add("X-Amz-Acl", cannedACL) - return &objInfo, nil - } - - grantACL := getAmzGrantACL(res) - for k, v := range grantACL { - objInfo.Metadata[k] = v - } - - return &objInfo, nil -} - -func getCannedACL(aCPolicy *accessControlPolicy) string { - grants := aCPolicy.AccessControlList.Grant - - switch { - case len(grants) == 1: - if grants[0].Grantee.URI == "" && grants[0].Permission == "FULL_CONTROL" { - return "private" - } - case len(grants) == 2: - for _, g := range grants { - if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" && g.Permission == "READ" { - return "authenticated-read" - } - if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && g.Permission == "READ" { - return "public-read" - } - if g.Permission == "READ" && g.Grantee.ID == aCPolicy.Owner.ID { - return "bucket-owner-read" - } - } - case len(grants) == 3: - for _, g := range grants { - if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && g.Permission == "WRITE" { - return "public-read-write" - } - } - } - return "" -} - -func getAmzGrantACL(aCPolicy *accessControlPolicy) map[string][]string { - grants := aCPolicy.AccessControlList.Grant - res := map[string][]string{} - - for _, g := range grants { - switch { - case g.Permission == "READ": - res["X-Amz-Grant-Read"] = append(res["X-Amz-Grant-Read"], "id="+g.Grantee.ID) - case g.Permission == "WRITE": - res["X-Amz-Grant-Write"] = append(res["X-Amz-Grant-Write"], "id="+g.Grantee.ID) - case g.Permission == "READ_ACP": - res["X-Amz-Grant-Read-Acp"] = append(res["X-Amz-Grant-Read-Acp"], "id="+g.Grantee.ID) - case g.Permission == "WRITE_ACP": - res["X-Amz-Grant-Write-Acp"] = append(res["X-Amz-Grant-Write-Acp"], "id="+g.Grantee.ID) - case g.Permission == "FULL_CONTROL": - res["X-Amz-Grant-Full-Control"] = append(res["X-Amz-Grant-Full-Control"], "id="+g.Grantee.ID) - } - } - return res -} diff --git a/api-get-object-acl.go b/api-get-object-acl.go index 9a664f189..86a30bb73 100644 --- a/api-get-object-acl.go +++ b/api-get-object-acl.go @@ -19,9 +19,122 @@ package minio import ( "context" + "net/http" + "net/url" ) -//GetObjectACL get object ACLs -func (c Client) GetObjectACL(bucketName, objectName string) (*ObjectInfo, error) { - return c.GetObjectACLWithContext(context.Background(), bucketName, objectName) +type accessControlPolicy struct { + Owner struct { + ID string `xml:"ID"` + DisplayName string `xml:"DisplayName"` + } `xml:"Owner"` + AccessControlList struct { + Grant []struct { + Grantee struct { + ID string `xml:"ID"` + DisplayName string `xml:"DisplayName"` + URI string `xml:"URI"` + } `xml:"Grantee"` + Permission string `xml:"Permission"` + } `xml:"Grant"` + } `xml:"AccessControlList"` +} + +// GetObjectACL get object ACLs +func (c Client) GetObjectACL(ctx context.Context, bucketName, objectName string) (*ObjectInfo, error) { + resp, err := c.executeMethod(ctx, "GET", requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: url.Values{ + "acl": []string{""}, + }, + }) + if err != nil { + return nil, err + } + defer closeResponse(resp) + + if resp.StatusCode != http.StatusOK { + return nil, httpRespToErrorResponse(resp, bucketName, objectName) + } + + res := &accessControlPolicy{} + + if err := xmlDecoder(resp.Body, res); err != nil { + return nil, err + } + + objInfo, err := c.statObject(ctx, bucketName, objectName, StatObjectOptions{}) + if err != nil { + return nil, err + } + + objInfo.Owner.DisplayName = res.Owner.DisplayName + objInfo.Owner.ID = res.Owner.ID + + objInfo.Grant = append(objInfo.Grant, res.AccessControlList.Grant...) + + cannedACL := getCannedACL(res) + if cannedACL != "" { + objInfo.Metadata.Add("X-Amz-Acl", cannedACL) + return &objInfo, nil + } + + grantACL := getAmzGrantACL(res) + for k, v := range grantACL { + objInfo.Metadata[k] = v + } + + return &objInfo, nil +} + +func getCannedACL(aCPolicy *accessControlPolicy) string { + grants := aCPolicy.AccessControlList.Grant + + switch { + case len(grants) == 1: + if grants[0].Grantee.URI == "" && grants[0].Permission == "FULL_CONTROL" { + return "private" + } + case len(grants) == 2: + for _, g := range grants { + if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" && g.Permission == "READ" { + return "authenticated-read" + } + if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && g.Permission == "READ" { + return "public-read" + } + if g.Permission == "READ" && g.Grantee.ID == aCPolicy.Owner.ID { + return "bucket-owner-read" + } + } + case len(grants) == 3: + for _, g := range grants { + if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && g.Permission == "WRITE" { + return "public-read-write" + } + } + } + return "" +} + +func getAmzGrantACL(aCPolicy *accessControlPolicy) map[string][]string { + grants := aCPolicy.AccessControlList.Grant + res := map[string][]string{} + + for _, g := range grants { + switch { + case g.Permission == "READ": + res["X-Amz-Grant-Read"] = append(res["X-Amz-Grant-Read"], "id="+g.Grantee.ID) + case g.Permission == "WRITE": + res["X-Amz-Grant-Write"] = append(res["X-Amz-Grant-Write"], "id="+g.Grantee.ID) + case g.Permission == "READ_ACP": + res["X-Amz-Grant-Read-Acp"] = append(res["X-Amz-Grant-Read-Acp"], "id="+g.Grantee.ID) + case g.Permission == "WRITE_ACP": + res["X-Amz-Grant-Write-Acp"] = append(res["X-Amz-Grant-Write-Acp"], "id="+g.Grantee.ID) + case g.Permission == "FULL_CONTROL": + res["X-Amz-Grant-Full-Control"] = append(res["X-Amz-Grant-Full-Control"], "id="+g.Grantee.ID) + } + } + return res } diff --git a/api-get-object-context.go b/api-get-object-context.go deleted file mode 100644 index 3c3afd059..000000000 --- a/api-get-object-context.go +++ /dev/null @@ -1,26 +0,0 @@ -/* - * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import "context" - -// GetObjectWithContext - returns an seekable, readable object. -// The options can be used to specify the GET request further. -func (c Client) GetObjectWithContext(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (*Object, error) { - return c.getObjectWithContext(ctx, bucketName, objectName, opts) -} diff --git a/api-get-object-file.go b/api-get-object-file.go index c2d277514..464854ae3 100644 --- a/api-get-object-file.go +++ b/api-get-object-file.go @@ -23,22 +23,12 @@ import ( "os" "path/filepath" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) -// FGetObjectWithContext - download contents of an object to a local file. -// The options can be used to specify the GET request further. -func (c Client) FGetObjectWithContext(ctx context.Context, bucketName, objectName, filePath string, opts GetObjectOptions) error { - return c.fGetObjectWithContext(ctx, bucketName, objectName, filePath, opts) -} - // FGetObject - download contents of an object to a local file. -func (c Client) FGetObject(bucketName, objectName, filePath string, opts GetObjectOptions) error { - return c.fGetObjectWithContext(context.Background(), bucketName, objectName, filePath, opts) -} - -// fGetObjectWithContext - fgetObject wrapper function with context -func (c Client) fGetObjectWithContext(ctx context.Context, bucketName, objectName, filePath string, opts GetObjectOptions) error { +// The options can be used to specify the GET request further. +func (c Client) FGetObject(ctx context.Context, bucketName, objectName, filePath string, opts GetObjectOptions) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -73,7 +63,7 @@ func (c Client) fGetObjectWithContext(ctx context.Context, bucketName, objectNam } // Gather md5sum. - objectStat, err := c.StatObject(bucketName, objectName, StatObjectOptions{opts}) + objectStat, err := c.StatObject(ctx, bucketName, objectName, StatObjectOptions{opts}) if err != nil { return err } diff --git a/api-get-object.go b/api-get-object.go index 14370b0cd..e88a85e54 100644 --- a/api-get-object.go +++ b/api-get-object.go @@ -25,16 +25,11 @@ import ( "net/http" "sync" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) -// GetObject - returns an seekable, readable object. -func (c Client) GetObject(bucketName, objectName string, opts GetObjectOptions) (*Object, error) { - return c.getObjectWithContext(context.Background(), bucketName, objectName, opts) -} - // GetObject wrapper function that accepts a request context -func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (*Object, error) { +func (c Client) GetObject(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (*Object, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return nil, err diff --git a/api-get-options.go b/api-get-options.go index 538fd1a05..ff38ba72b 100644 --- a/api-get-options.go +++ b/api-get-options.go @@ -22,7 +22,7 @@ import ( "net/http" "time" - "github.com/minio/minio-go/v6/pkg/encrypt" + "github.com/minio/minio-go/v7/pkg/encrypt" ) // GetObjectOptions are used to specify additional headers or options diff --git a/api-get-policy.go b/api-get-policy.go index 590b66c4b..859c63901 100644 --- a/api-get-policy.go +++ b/api-get-policy.go @@ -23,16 +23,11 @@ import ( "net/http" "net/url" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) -// GetBucketPolicy is a wrapper for GetBucketPolicyWithContext -func (c Client) GetBucketPolicy(bucketName string) (string, error) { - return c.GetBucketPolicyWithContext(context.Background(), bucketName) -} - -// GetBucketPolicyWithContext - get bucket policy at a given path. -func (c Client) GetBucketPolicyWithContext(ctx context.Context, bucketName string) (string, error) { +// GetBucketPolicy returns the current policy +func (c Client) GetBucketPolicy(ctx context.Context, bucketName string) (string, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return "", err diff --git a/api-list.go b/api-list.go index 7823181f3..8aea481eb 100644 --- a/api-list.go +++ b/api-list.go @@ -23,7 +23,7 @@ import ( "net/http" "net/url" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) // ListBuckets list all buckets owned by this authenticated user. @@ -32,26 +32,11 @@ import ( // allowed for listing buckets. // // api := client.New(....) -// for message := range api.ListBuckets() { +// for message := range api.ListBuckets(context.Background()) { // fmt.Println(message) // } // -func (c Client) ListBuckets() ([]BucketInfo, error) { - return c.ListBucketsWithContext(context.Background()) -} - -// ListBucketsWithContext list all buckets owned by this authenticated user, -// accepts a context for facilitate cancellation. -// -// This call requires explicit authentication, no anonymous requests are -// allowed for listing buckets. -// -// api := client.New(....) -// for message := range api.ListBucketsWithContext(context.Background()) { -// fmt.Println(message) -// } -// -func (c Client) ListBucketsWithContext(ctx context.Context) ([]BucketInfo, error) { +func (c Client) ListBuckets(ctx context.Context) ([]BucketInfo, error) { // Execute GET on service. resp, err := c.executeMethod(ctx, "GET", requestMetadata{contentSHA256Hex: emptySHA256Hex}) defer closeResponse(resp) @@ -73,14 +58,7 @@ func (c Client) ListBucketsWithContext(ctx context.Context) ([]BucketInfo, error /// Bucket Read Operations. -// ListObjectsV2WithMetadata is the wrapper for ListObjectsV2WithMetadataWithContext -func (c Client) ListObjectsV2WithMetadata(bucketName, objectPrefix string, - recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { - return c.ListObjectsV2WithMetadataWithContext(context.Background(), - bucketName, objectPrefix, recursive, doneCh) -} - -// ListObjectsV2WithMetadataWithContext lists all objects matching the objectPrefix +// ListObjectsV2WithMetadata lists all objects matching the objectPrefix // from the specified bucket. If recursion is enabled it would list // all subdirectories and all its contents. This call adds // UserMetadata information as well for each object. @@ -102,17 +80,16 @@ func (c Client) ListObjectsV2WithMetadata(bucketName, objectPrefix string, // recursive := true // // Add metadata // metadata := true -// for message := range api.ListObjectsV2WithMetadataWithContext(ctx, -// "mytestbucket", "starthere", recursive, doneCh) { +// for message := range api.ListObjectsV2WithMetadata(ctx, "mytestbucket", "starthere", recursive, doneCh) { // fmt.Println(message) // } // -func (c Client) ListObjectsV2WithMetadataWithContext(ctx context.Context, bucketName, objectPrefix string, recursive bool, - doneCh <-chan struct{}) <-chan ObjectInfo { +func (c Client) ListObjectsV2WithMetadata(ctx context.Context, bucketName, objectPrefix string, + recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { // Check whether this is snowball region, if yes ListObjectsV2 doesn't work, fallback to listObjectsV1. if location, ok := c.bucketLocCache.Get(bucketName); ok { if location == "snowball" { - return c.ListObjects(bucketName, objectPrefix, recursive, doneCh) + return c.ListObjects(ctx, bucketName, objectPrefix, recursive, doneCh) } } return c.listObjectsV2(ctx, bucketName, objectPrefix, recursive, true, doneCh) @@ -219,20 +196,15 @@ func (c Client) listObjectsV2(ctx context.Context, bucketName, objectPrefix stri // defer close(doneCh) // // Recursively list all objects in 'mytestbucket' // recursive := true -// for message := range api.ListObjectsV2("mytestbucket", "starthere", recursive, doneCh) { +// for message := range api.ListObjectsV2(ctx, "mytestbucket", "starthere", recursive, doneCh) { // fmt.Println(message) // } // -func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { - return c.ListObjectsV2WithContext(context.Background(), bucketName, objectPrefix, recursive, doneCh) -} - -// ListObjectsV2WithContext does the ListObjectsV2 job with context -func (c Client) ListObjectsV2WithContext(ctx context.Context, bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { +func (c Client) ListObjectsV2(ctx context.Context, bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { // Check whether this is snowball region, if yes ListObjectsV2 doesn't work, fallback to listObjectsV1. if location, ok := c.bucketLocCache.Get(bucketName); ok { if location == "snowball" { - return c.ListObjectsWithContext(ctx, bucketName, objectPrefix, recursive, doneCh) + return c.ListObjects(ctx, bucketName, objectPrefix, recursive, doneCh) } } return c.listObjectsV2(ctx, bucketName, objectPrefix, recursive, false, doneCh) @@ -347,16 +319,9 @@ func (c Client) listObjectsV2Query(ctx context.Context, bucketName, objectPrefix return listBucketResult, nil } -// ListObjects is a wrapper for ListObjectsWithContext -func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, - doneCh <-chan struct{}) <-chan ObjectInfo { - return c.ListObjectsWithContext(context.Background(), bucketName, - objectPrefix, recursive, doneCh) -} - -// ListObjectsWithContext - (List Objects with context) - List some +// ListObjects - (List Objects with context) - List some // objects or all recursively. -// ListObjectsWithContext lists all objects matching the objectPrefix from +// ListObjects lists all objects matching the objectPrefix from // the specified bucket. If recursion is enabled it would list // all subdirectories and all its contents. // @@ -372,13 +337,13 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, // defer close(doneCh) // // Recurively list all objects in 'mytestbucket' // recursive := true -// for message := range api.ListObjectsWithContext(ctx, "mytestbucket", +// for message := range api.ListObjects(ctx, "mytestbucket", // "starthere", recursive, doneCh) { // fmt.Println(message) // } // -func (c Client) ListObjectsWithContext(ctx context.Context, bucketName, - objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { +func (c Client) ListObjects(ctx context.Context, bucketName, objectPrefix string, recursive bool, + doneCh <-chan struct{}) <-chan ObjectInfo { // Allocate new list objects channel. objectStatCh := make(chan ObjectInfo, 1) // Default listing is delimited at "/" @@ -567,19 +532,12 @@ func (c Client) listObjectsQuery(ctx context.Context, bucketName, objectPrefix, // } // ListIncompleteUploads is a wrapper for ListIncompleteUploadsWithContent -func (c Client) ListIncompleteUploads(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo { - return c.ListIncompleteUploadsWithContext(context.Background(), bucketName, objectPrefix, recursive, doneCh) -} - -// ListIncompleteUploadsWithContext adds the context argument to ListIncompleteUploads functionality -func (c Client) ListIncompleteUploadsWithContext(ctx context.Context, bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo { - // Turn on size aggregation of individual parts. - isAggregateSize := true - return c.listIncompleteUploads(ctx, bucketName, objectPrefix, recursive, isAggregateSize, doneCh) +func (c Client) ListIncompleteUploads(ctx context.Context, bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo { + return c.listIncompleteUploads(ctx, bucketName, objectPrefix, recursive, doneCh) } // listIncompleteUploads lists all incomplete uploads. -func (c Client) listIncompleteUploads(ctx context.Context, bucketName, objectPrefix string, recursive, aggregateSize bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo { +func (c Client) listIncompleteUploads(ctx context.Context, bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo { // Allocate channel for multipart uploads. objectMultipartStatCh := make(chan ObjectMultipartInfo, 1) // Delimiter is set to "/" by default. @@ -624,16 +582,6 @@ func (c Client) listIncompleteUploads(ctx context.Context, bucketName, objectPre // Send all multipart uploads. for _, obj := range result.Uploads { // Calculate total size of the uploaded parts if 'aggregateSize' is enabled. - if aggregateSize { - // Get total multipart size. - obj.Size, err = c.getTotalMultipartSize(ctx, bucketName, obj.Key, obj.UploadID) - if err != nil { - objectMultipartStatCh <- ObjectMultipartInfo{ - Err: err, - } - continue - } - } select { // Send individual uploads here. case objectMultipartStatCh <- obj: @@ -787,13 +735,11 @@ func (c Client) findUploadIDs(ctx context.Context, bucketName, objectName string var uploadIDs []string // Make list incomplete uploads recursive. isRecursive := true - // Turn off size aggregation of individual parts, in this request. - isAggregateSize := false // Create done channel to cleanup the routine. doneCh := make(chan struct{}) defer close(doneCh) // List all incomplete uploads. - for mpUpload := range c.listIncompleteUploads(ctx, bucketName, objectName, isRecursive, isAggregateSize, doneCh) { + for mpUpload := range c.listIncompleteUploads(ctx, bucketName, objectName, isRecursive, doneCh) { if mpUpload.Err != nil { return nil, mpUpload.Err } @@ -805,18 +751,6 @@ func (c Client) findUploadIDs(ctx context.Context, bucketName, objectName string return uploadIDs, nil } -func (c Client) getTotalMultipartSize(ctx context.Context, bucketName, objectName, uploadID string) (size int64, err error) { - // Iterate over all parts and aggregate the size. - partsInfo, err := c.listObjectParts(ctx, bucketName, objectName, uploadID) - if err != nil { - return 0, err - } - for _, partInfo := range partsInfo { - size += partInfo.Size - } - return size, nil -} - // listObjectPartsQuery (List Parts query) // - lists some or all (up to 1000) parts that have been uploaded // for a specific multipart upload diff --git a/api-notification.go b/api-notification.go index 624468e2a..0d91d7800 100644 --- a/api-notification.go +++ b/api-notification.go @@ -25,16 +25,11 @@ import ( "time" jsoniter "github.com/json-iterator/go" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) -// GetBucketNotification is a wrapper for GetBucketNotificationWithContext -func (c Client) GetBucketNotification(bucketName string) (bucketNotification BucketNotification, err error) { - return c.GetBucketNotificationWithContext(context.Background(), bucketName) -} - -// GetBucketNotificationWithContext - get bucket notification at a given path. -func (c Client) GetBucketNotificationWithContext(ctx context.Context, bucketName string) (bucketNotification BucketNotification, err error) { +// GetBucketNotification returns current bucket notification configuration +func (c Client) GetBucketNotification(ctx context.Context, bucketName string) (bucketNotification BucketNotification, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return BucketNotification{}, err @@ -140,13 +135,8 @@ type NotificationInfo struct { Err error } -// ListenBucketNotification is a wrapper for ListenBucketNotificationWithContext. -func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, events []string, doneCh <-chan struct{}) <-chan NotificationInfo { - return c.ListenBucketNotificationWithContext(context.Background(), bucketName, prefix, suffix, events, doneCh) -} - -// ListenBucketNotificationWithContext - listen on bucket notifications. -func (c Client) ListenBucketNotificationWithContext(ctx context.Context, bucketName, prefix, suffix string, events []string, doneCh <-chan struct{}) <-chan NotificationInfo { +// ListenBucketNotification listen for bucket events, this is a MinIO specific API +func (c Client) ListenBucketNotification(ctx context.Context, bucketName, prefix, suffix string, events []string, doneCh <-chan struct{}) <-chan NotificationInfo { notificationInfoCh := make(chan NotificationInfo, 1) const notificationCapacity = 1024 * 1024 notificationEventBuffer := make([]byte, notificationCapacity) diff --git a/api-object-legal-hold.go b/api-object-legal-hold.go index 82665b235..8dbf4b04c 100644 --- a/api-object-legal-hold.go +++ b/api-object-legal-hold.go @@ -25,7 +25,7 @@ import ( "net/http" "net/url" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) // objectLegalHold - object legal hold specified in @@ -80,13 +80,8 @@ func newObjectLegalHold(status *LegalHoldStatus) (*objectLegalHold, error) { return legalHold, nil } -// PutObjectLegalHold is the wrapper for PutObjectLegalHoldWithContext. -func (c Client) PutObjectLegalHold(bucketName, objectName string, opts PutObjectLegalHoldOptions) error { - return c.PutObjectLegalHoldWithContext(context.Background(), bucketName, objectName, opts) -} - -// PutObjectLegalHoldWithContext : sets object legal hold for a given object and versionID. -func (c Client) PutObjectLegalHoldWithContext(ctx context.Context, bucketName, objectName string, opts PutObjectLegalHoldOptions) error { +// PutObjectLegalHold : sets object legal hold for a given object and versionID. +func (c Client) PutObjectLegalHold(ctx context.Context, bucketName, objectName string, opts PutObjectLegalHoldOptions) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -140,7 +135,7 @@ func (c Client) PutObjectLegalHoldWithContext(ctx context.Context, bucketName, o } // GetObjectLegalHold gets legal-hold status of given object. -func (c Client) GetObjectLegalHold(bucketName, objectName string, opts GetObjectLegalHoldOptions) (status *LegalHoldStatus, err error) { +func (c Client) GetObjectLegalHold(ctx context.Context, bucketName, objectName string, opts GetObjectLegalHoldOptions) (status *LegalHoldStatus, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return nil, err @@ -157,7 +152,7 @@ func (c Client) GetObjectLegalHold(bucketName, objectName string, opts GetObject } // Execute GET on bucket to list objects. - resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, "GET", requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, diff --git a/api-object-lock.go b/api-object-lock.go index 0a40206f9..4b5f72dc2 100644 --- a/api-object-lock.go +++ b/api-object-lock.go @@ -26,7 +26,7 @@ import ( "net/url" "time" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) // RetentionMode - object retention mode. @@ -138,13 +138,8 @@ func newObjectLockConfig(mode *RetentionMode, validity *uint, unit *ValidityUnit return nil, fmt.Errorf("all of retention mode, validity and validity unit must be passed") } -// SetBucketObjectLockConfig is a wrapper for SetBucketObjectLockConfigWithContext. -func (c Client) SetBucketObjectLockConfig(bucketName string, mode *RetentionMode, validity *uint, unit *ValidityUnit) error { - return c.SetBucketObjectLockConfigWithContext(context.Background(), bucketName, mode, validity, unit) -} - -// SetBucketObjectLockConfigWithContext sets object lock configuration in given bucket. mode, validity and unit are either all set or all nil. -func (c Client) SetBucketObjectLockConfigWithContext(ctx context.Context, bucketName string, mode *RetentionMode, validity *uint, unit *ValidityUnit) error { +// SetBucketObjectLockConfig sets object lock configuration in given bucket. mode, validity and unit are either all set or all nil. +func (c Client) SetBucketObjectLockConfig(ctx context.Context, bucketName string, mode *RetentionMode, validity *uint, unit *ValidityUnit) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -188,13 +183,8 @@ func (c Client) SetBucketObjectLockConfigWithContext(ctx context.Context, bucket return nil } -// GetObjectLockConfig is a wrapper for GetObjectLockConfigWithContext -func (c Client) GetObjectLockConfig(bucketName string) (objectLock string, mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { - return c.GetObjectLockConfigWithContext(context.Background(), bucketName) -} - -// GetObjectLockConfigWithContext gets object lock configuration of given bucket. -func (c Client) GetObjectLockConfigWithContext(ctx context.Context, bucketName string) (objectLock string, mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { +// GetObjectLockConfig gets object lock configuration of given bucket. +func (c Client) GetObjectLockConfig(ctx context.Context, bucketName string) (objectLock string, mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return "", nil, nil, nil, err @@ -240,18 +230,12 @@ func (c Client) GetObjectLockConfigWithContext(ctx context.Context, bucketName s } // GetBucketObjectLockConfig gets object lock configuration of given bucket. -func (c Client) GetBucketObjectLockConfig(bucketName string) ( - mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { - return c.GetBucketObjectLockConfigWithContext(context.Background(), bucketName) -} - -// GetBucketObjectLockConfigWithContext gets object lock configuration of given bucket. -func (c Client) GetBucketObjectLockConfigWithContext(ctx context.Context, bucketName string) (mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { - _, mode, validity, unit, err = c.GetObjectLockConfigWithContext(ctx, bucketName) +func (c Client) GetBucketObjectLockConfig(ctx context.Context, bucketName string) (mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { + _, mode, validity, unit, err = c.GetObjectLockConfig(ctx, bucketName) return mode, validity, unit, err } // SetObjectLockConfig sets object lock configuration in given bucket. mode, validity and unit are either all set or all nil. -func (c Client) SetObjectLockConfig(bucketName string, mode *RetentionMode, validity *uint, unit *ValidityUnit) error { - return c.SetBucketObjectLockConfig(bucketName, mode, validity, unit) +func (c Client) SetObjectLockConfig(ctx context.Context, bucketName string, mode *RetentionMode, validity *uint, unit *ValidityUnit) error { + return c.SetBucketObjectLockConfig(ctx, bucketName, mode, validity, unit) } diff --git a/api-object-retention.go b/api-object-retention.go index 264f1e989..24c31af0d 100644 --- a/api-object-retention.go +++ b/api-object-retention.go @@ -26,7 +26,7 @@ import ( "net/url" "time" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) // objectRetention - object retention specified in @@ -65,13 +65,8 @@ type PutObjectRetentionOptions struct { VersionID string } -// PutObjectRetention is a wrapper for PutObjectRetentionWithContext. -func (c Client) PutObjectRetention(bucketName, objectName string, opts PutObjectRetentionOptions) error { - return c.PutObjectRetentionWithContext(context.Background(), bucketName, objectName, opts) -} - -// PutObjectRetentionWithContext : sets object retention for a given object and versionID. -func (c Client) PutObjectRetentionWithContext(ctx context.Context, bucketName, objectName string, opts PutObjectRetentionOptions) error { +// PutObjectRetention sets object retention for a given object and versionID. +func (c Client) PutObjectRetention(ctx context.Context, bucketName, objectName string, opts PutObjectRetentionOptions) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -134,7 +129,7 @@ func (c Client) PutObjectRetentionWithContext(ctx context.Context, bucketName, o } // GetObjectRetention gets retention of given object. -func (c Client) GetObjectRetention(bucketName, objectName, versionID string) (mode *RetentionMode, retainUntilDate *time.Time, err error) { +func (c Client) GetObjectRetention(ctx context.Context, bucketName, objectName, versionID string) (mode *RetentionMode, retainUntilDate *time.Time, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return nil, nil, err @@ -149,7 +144,7 @@ func (c Client) GetObjectRetention(bucketName, objectName, versionID string) (mo urlValues.Set("versionId", versionID) } // Execute GET on bucket to list objects. - resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, "GET", requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, diff --git a/api-object-tagging.go b/api-object-tagging.go index 65df4f3e7..42b6f62a5 100644 --- a/api-object-tagging.go +++ b/api-object-tagging.go @@ -25,18 +25,13 @@ import ( "net/http" "net/url" - "github.com/minio/minio-go/v6/pkg/s3utils" - "github.com/minio/minio-go/v6/pkg/tags" + "github.com/minio/minio-go/v7/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/tags" ) -// PutObjectTagging replaces or creates object tag(s) -func (c Client) PutObjectTagging(bucketName, objectName string, objectTags map[string]string) error { - return c.PutObjectTaggingWithContext(context.Background(), bucketName, objectName, objectTags) -} - -// PutObjectTaggingWithContext replaces or creates object tag(s) with a context to control cancellations +// PutObjectTagging replaces or creates object tag(s) with a context to control cancellations // and timeouts. -func (c Client) PutObjectTaggingWithContext(ctx context.Context, bucketName, objectName string, objectTags map[string]string) error { +func (c Client) PutObjectTagging(ctx context.Context, bucketName, objectName string, objectTags map[string]string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -80,14 +75,9 @@ func (c Client) PutObjectTaggingWithContext(ctx context.Context, bucketName, obj return nil } -// GetObjectTagging fetches object tag(s) -func (c Client) GetObjectTagging(bucketName, objectName string) (string, error) { - return c.GetObjectTaggingWithContext(context.Background(), bucketName, objectName) -} - -// GetObjectTaggingWithContext fetches object tag(s) with a context to control cancellations +// GetObjectTagging fetches object tag(s) with a context to control cancellations // and timeouts. -func (c Client) GetObjectTaggingWithContext(ctx context.Context, bucketName, objectName string) (string, error) { +func (c Client) GetObjectTagging(ctx context.Context, bucketName, objectName string) (string, error) { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) @@ -119,14 +109,9 @@ func (c Client) GetObjectTaggingWithContext(ctx context.Context, bucketName, obj return string(tagBuf), err } -// RemoveObjectTagging deletes object tag(s) -func (c Client) RemoveObjectTagging(bucketName, objectName string) error { - return c.RemoveObjectTaggingWithContext(context.Background(), bucketName, objectName) -} - -// RemoveObjectTaggingWithContext removes object tag(s) with a context to control cancellations +// RemoveObjectTagging removes object tag(s) with a context to control cancellations // and timeouts. -func (c Client) RemoveObjectTaggingWithContext(ctx context.Context, bucketName, objectName string) error { +func (c Client) RemoveObjectTagging(ctx context.Context, bucketName, objectName string) error { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) diff --git a/api-presigned.go b/api-presigned.go index ad84e3fa7..6e8958dd2 100644 --- a/api-presigned.go +++ b/api-presigned.go @@ -24,8 +24,8 @@ import ( "net/url" "time" - "github.com/minio/minio-go/v6/pkg/s3utils" - "github.com/minio/minio-go/v6/pkg/signer" + "github.com/minio/minio-go/v7/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/signer" ) // presignURL - Returns a presigned URL for an input 'method'. @@ -55,76 +55,53 @@ func (c Client) presignURL(ctx context.Context, method string, bucketName string // Instantiate a new request. // Since expires is set newRequest will presign the request. var req *http.Request - if req, err = c.newRequest(method, reqMetadata); err != nil { + if req, err = c.newRequest(ctx, method, reqMetadata); err != nil { return nil, err } return req.URL, nil } -// PresignedGetObject is a wrapper for PresignedGetObjectWithContext -func (c Client) PresignedGetObject(bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { - return c.PresignedGetObjectWithContext(context.Background(), bucketName, objectName, expires, reqParams) -} - -// PresignedGetObjectWithContext - Returns a presigned URL to access an object +// PresignedGetObject - Returns a presigned URL to access an object // data without credentials. URL can have a maximum expiry of // upto 7days or a minimum of 1sec. Additionally you can override // a set of response headers using the query parameters. -func (c Client) PresignedGetObjectWithContext(ctx context.Context, bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { +func (c Client) PresignedGetObject(ctx context.Context, bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { if err = s3utils.CheckValidObjectName(objectName); err != nil { return nil, err } return c.presignURL(ctx, "GET", bucketName, objectName, expires, reqParams) } -// PresignedHeadObject is the wrapper for PresignedHeadObjectWithContext -func (c Client) PresignedHeadObject(bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { - if err = s3utils.CheckValidObjectName(objectName); err != nil { - return nil, err - } - return c.PresignedHeadObjectWithContext(context.Background(), bucketName, objectName, expires, reqParams) -} - -// PresignedHeadObjectWithContext - Returns a presigned URL to access +// PresignedHeadObject - Returns a presigned URL to access // object metadata without credentials. URL can have a maximum expiry // of upto 7days or a minimum of 1sec. Additionally you can override // a set of response headers using the query parameters. -func (c Client) PresignedHeadObjectWithContext(ctx context.Context, bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { +func (c Client) PresignedHeadObject(ctx context.Context, bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { if err = s3utils.CheckValidObjectName(objectName); err != nil { return nil, err } return c.presignURL(ctx, "HEAD", bucketName, objectName, expires, reqParams) } -// PresignedPutObject is a wrapper for PresignedPutObjectWithContext -func (c Client) PresignedPutObject(bucketName string, objectName string, expires time.Duration) (u *url.URL, err error) { - return c.PresignedPutObjectWithContext(context.Background(), bucketName, objectName, expires) -} - -// PresignedPutObjectWithContext - Returns a presigned URL to upload an object +// PresignedPutObject - Returns a presigned URL to upload an object // without credentials. URL can have a maximum expiry of upto 7days // or a minimum of 1sec. -func (c Client) PresignedPutObjectWithContext(ctx context.Context, bucketName string, objectName string, expires time.Duration) (u *url.URL, err error) { +func (c Client) PresignedPutObject(ctx context.Context, bucketName string, objectName string, expires time.Duration) (u *url.URL, err error) { if err = s3utils.CheckValidObjectName(objectName); err != nil { return nil, err } return c.presignURL(ctx, "PUT", bucketName, objectName, expires, nil) } -// Presign is a wrapper for PresignWithContext -func (c Client) Presign(method string, bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { - return c.presignURL(context.Background(), method, bucketName, objectName, expires, reqParams) -} - -// PresignWithContext - returns a presigned URL for any http method of your choice +// Presign - returns a presigned URL for any http method of your choice // along with custom request params. URL can have a maximum expiry of // upto 7days or a minimum of 1sec. -func (c Client) PresignWithContext(ctx context.Context, method string, bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { +func (c Client) Presign(ctx context.Context, method string, bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { return c.presignURL(ctx, method, bucketName, objectName, expires, reqParams) } // PresignedPostPolicy - Returns POST urlString, form data to upload an object. -func (c Client) PresignedPostPolicy(p *PostPolicy) (u *url.URL, formData map[string]string, err error) { +func (c Client) PresignedPostPolicy(ctx context.Context, p *PostPolicy) (u *url.URL, formData map[string]string, err error) { // Validate input arguments. if p.expiration.IsZero() { return nil, nil, errors.New("Expiration time must be specified") @@ -138,7 +115,7 @@ func (c Client) PresignedPostPolicy(p *PostPolicy) (u *url.URL, formData map[str bucketName := p.formData["bucket"] // Fetch the bucket location. - location, err := c.getBucketLocation(bucketName) + location, err := c.getBucketLocation(ctx, bucketName) if err != nil { return nil, nil, err } diff --git a/api-put-bucket.go b/api-put-bucket.go index 959d6ed21..6db66c756 100644 --- a/api-put-bucket.go +++ b/api-put-bucket.go @@ -26,7 +26,7 @@ import ( "net/url" "strings" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) // ApplyServerSideEncryptionByDefault defines default encryption configuration, KMS or SSE. To activate @@ -126,25 +126,14 @@ func (c Client) doMakeBucket(ctx context.Context, bucketName string, location st return nil } -// MakeBucket creates a new bucket with bucketName. +// MakeBucket creates a new bucket with bucketName with a context to control cancellations and timeouts. // // Location is an optional argument, by default all buckets are // created in US Standard Region. // // For Amazon S3 for more supported regions - http://docs.aws.amazon.com/general/latest/gr/rande.html // For Google Cloud Storage for more supported regions - https://cloud.google.com/storage/docs/bucket-locations -func (c Client) MakeBucket(bucketName string, location string) (err error) { - return c.MakeBucketWithContext(context.Background(), bucketName, location) -} - -// MakeBucketWithContext creates a new bucket with bucketName with a context to control cancellations and timeouts. -// -// Location is an optional argument, by default all buckets are -// created in US Standard Region. -// -// For Amazon S3 for more supported regions - http://docs.aws.amazon.com/general/latest/gr/rande.html -// For Google Cloud Storage for more supported regions - https://cloud.google.com/storage/docs/bucket-locations -func (c Client) MakeBucketWithContext(ctx context.Context, bucketName string, location string) (err error) { +func (c Client) MakeBucket(ctx context.Context, bucketName string, location string) (err error) { return c.makeBucket(ctx, bucketName, location, false) } @@ -155,29 +144,12 @@ func (c Client) MakeBucketWithContext(ctx context.Context, bucketName string, lo // // For Amazon S3 for more supported regions - http://docs.aws.amazon.com/general/latest/gr/rande.html // For Google Cloud Storage for more supported regions - https://cloud.google.com/storage/docs/bucket-locations -func (c Client) MakeBucketWithObjectLock(bucketName string, location string) (err error) { - return c.MakeBucketWithObjectLockWithContext(context.Background(), bucketName, location) -} - -// MakeBucketWithObjectLockWithContext creates a object lock enabled new bucket with bucketName with a context to -// control cancellations and timeouts. -// -// Location is an optional argument, by default all buckets are -// created in US Standard Region. -// -// For Amazon S3 for more supported regions - http://docs.aws.amazon.com/general/latest/gr/rande.html -// For Google Cloud Storage for more supported regions - https://cloud.google.com/storage/docs/bucket-locations -func (c Client) MakeBucketWithObjectLockWithContext(ctx context.Context, bucketName string, location string) (err error) { +func (c Client) MakeBucketWithObjectLock(ctx context.Context, bucketName string, location string) (err error) { return c.makeBucket(ctx, bucketName, location, true) } // SetBucketPolicy sets the access permissions on an existing bucket. -func (c Client) SetBucketPolicy(bucketName, policy string) error { - return c.SetBucketPolicyWithContext(context.Background(), bucketName, policy) -} - -// SetBucketPolicyWithContext sets the access permissions on an existing bucket. -func (c Client) SetBucketPolicyWithContext(ctx context.Context, bucketName, policy string) error { +func (c Client) SetBucketPolicy(ctx context.Context, bucketName, policy string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -257,12 +229,7 @@ func (c Client) removeBucketPolicy(ctx context.Context, bucketName string) error } // SetBucketLifecycle set the lifecycle on an existing bucket. -func (c Client) SetBucketLifecycle(bucketName, lifecycle string) error { - return c.SetBucketLifecycleWithContext(context.Background(), bucketName, lifecycle) -} - -// SetBucketLifecycleWithContext set the lifecycle on an existing bucket with a context to control cancellations and timeouts. -func (c Client) SetBucketLifecycleWithContext(ctx context.Context, bucketName, lifecycle string) error { +func (c Client) SetBucketLifecycle(ctx context.Context, bucketName, lifecycle string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -343,12 +310,7 @@ func (c Client) removeBucketLifecycle(ctx context.Context, bucketName string) er } // SetBucketEncryption sets the default encryption configuration on an existing bucket. -func (c Client) SetBucketEncryption(bucketName string, configuration ServerSideEncryptionConfiguration) error { - return c.SetBucketEncryptionWithContext(context.Background(), bucketName, configuration) -} - -// SetBucketEncryptionWithContext sets the default encryption configuration on an existing bucket with a context to control cancellations and timeouts. -func (c Client) SetBucketEncryptionWithContext(ctx context.Context, bucketName string, configuration ServerSideEncryptionConfiguration) error { +func (c Client) SetBucketEncryption(ctx context.Context, bucketName string, configuration ServerSideEncryptionConfiguration) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -385,13 +347,8 @@ func (c Client) SetBucketEncryptionWithContext(ctx context.Context, bucketName s return nil } -// DeleteBucketEncryption removes the default encryption configuration on a bucket. -func (c Client) DeleteBucketEncryption(bucketName string) error { - return c.DeleteBucketEncryptionWithContext(context.Background(), bucketName) -} - -// DeleteBucketEncryptionWithContext removes the default encryption configuration on a bucket with a context to control cancellations and timeouts. -func (c Client) DeleteBucketEncryptionWithContext(ctx context.Context, bucketName string) error { +// DeleteBucketEncryption removes the default encryption configuration on a bucket with a context to control cancellations and timeouts. +func (c Client) DeleteBucketEncryption(ctx context.Context, bucketName string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -418,14 +375,9 @@ func (c Client) DeleteBucketEncryptionWithContext(ctx context.Context, bucketNam return nil } -// SetBucketNotification saves a new bucket notification. -func (c Client) SetBucketNotification(bucketName string, bucketNotification BucketNotification) error { - return c.SetBucketNotificationWithContext(context.Background(), bucketName, bucketNotification) -} - -// SetBucketNotificationWithContext saves a new bucket notification with a context to control cancellations +// SetBucketNotification saves a new bucket notification with a context to control cancellations // and timeouts. -func (c Client) SetBucketNotificationWithContext(ctx context.Context, bucketName string, bucketNotification BucketNotification) error { +func (c Client) SetBucketNotification(ctx context.Context, bucketName string, bucketNotification BucketNotification) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -465,14 +417,9 @@ func (c Client) SetBucketNotificationWithContext(ctx context.Context, bucketName return nil } -// RemoveAllBucketNotification is a wrapper for RemoveAllBucketNotificationWithContext -func (c Client) RemoveAllBucketNotification(bucketName string) error { - return c.RemoveAllBucketNotificationWithContext(context.Background(), bucketName) -} - -// RemoveAllBucketNotificationWithContext - Remove bucket notification clears all previously specified config -func (c Client) RemoveAllBucketNotificationWithContext(ctx context.Context, bucketName string) error { - return c.SetBucketNotificationWithContext(ctx, bucketName, BucketNotification{}) +// RemoveAllBucketNotification - Remove bucket notification clears all previously specified config +func (c Client) RemoveAllBucketNotification(ctx context.Context, bucketName string) error { + return c.SetBucketNotification(ctx, bucketName, BucketNotification{}) } var ( @@ -522,21 +469,11 @@ func (c Client) setVersioning(ctx context.Context, bucketName string, config []b } // EnableVersioning - Enable object versioning in given bucket. -func (c Client) EnableVersioning(bucketName string) error { - return c.EnableVersioningWithContext(context.Background(), bucketName) -} - -// EnableVersioningWithContext - Enable object versioning in given bucket with a context to control cancellations and timeouts. -func (c Client) EnableVersioningWithContext(ctx context.Context, bucketName string) error { +func (c Client) EnableVersioning(ctx context.Context, bucketName string) error { return c.setVersioning(ctx, bucketName, versionEnableConfig, versionEnableConfigLen, versionEnableConfigMD5Sum, versionEnableConfigSHA256) } // DisableVersioning - Disable object versioning in given bucket. -func (c Client) DisableVersioning(bucketName string) error { - return c.DisableVersioningWithContext(context.Background(), bucketName) -} - -// DisableVersioningWithContext - Disable object versioning in given bucket with a context to control cancellations and timeouts. -func (c Client) DisableVersioningWithContext(ctx context.Context, bucketName string) error { +func (c Client) DisableVersioning(ctx context.Context, bucketName string) error { return c.setVersioning(ctx, bucketName, versionDisableConfig, versionDisableConfigLen, versionDisableConfigMD5Sum, versionDisableConfigSHA256) } diff --git a/api-put-object-common.go b/api-put-object-common.go index 78e0a2fdc..d3ff8d023 100644 --- a/api-put-object-common.go +++ b/api-put-object-common.go @@ -23,7 +23,7 @@ import ( "math" "os" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) // Verify if reader is *minio.Object diff --git a/api-put-object-context.go b/api-put-object-context.go deleted file mode 100644 index 415a7878f..000000000 --- a/api-put-object-context.go +++ /dev/null @@ -1,33 +0,0 @@ -/* - * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "context" - "io" -) - -// PutObjectWithContext - Identical to PutObject call, but accepts context to facilitate request cancellation. -func (c Client) PutObjectWithContext(ctx context.Context, bucketName, objectName string, reader io.Reader, objectSize int64, - opts PutObjectOptions) (n int64, err error) { - err = opts.validate() - if err != nil { - return 0, err - } - return c.putObjectCommon(ctx, bucketName, objectName, reader, objectSize, opts) -} diff --git a/api-put-object-copy.go b/api-put-object-copy.go index d2f8b7f4b..227ddd1bb 100644 --- a/api-put-object-copy.go +++ b/api-put-object-copy.go @@ -24,24 +24,17 @@ import ( "net/http" "time" - "github.com/minio/minio-go/v6/pkg/encrypt" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/encrypt" + "github.com/minio/minio-go/v7/pkg/s3utils" ) // CopyObject - copy a source object into a new object -func (c Client) CopyObject(dst DestinationInfo, src SourceInfo) error { - return c.CopyObjectWithProgress(dst, src, nil) +func (c Client) CopyObject(ctx context.Context, dst DestinationInfo, src SourceInfo) error { + return c.CopyObjectWithProgress(ctx, dst, src, nil) } -// CopyObjectWithProgress is a wrapper for CopyObjectWithProgressWithContext -// progress bar input to notify current progress. -func (c Client) CopyObjectWithProgress(dst DestinationInfo, src SourceInfo, progress io.Reader) error { - return c.CopyObjectWithProgressWithContext(context.Background(), dst, src, progress) -} - -// CopyObjectWithProgressWithContext - copy a source object into a new object, optionally takes -// progress bar input to notify current progress. -func (c Client) CopyObjectWithProgressWithContext(ctx context.Context, dst DestinationInfo, src SourceInfo, progress io.Reader) error { +// CopyObjectWithProgress is like CopyObject with additional progress bar. +func (c Client) CopyObjectWithProgress(ctx context.Context, dst DestinationInfo, src SourceInfo, progress io.Reader) error { header := make(http.Header) for k, v := range src.Headers { header[k] = v diff --git a/api-put-object-file-context.go b/api-put-object-file-context.go index fb22c0d64..a55da8364 100644 --- a/api-put-object-file-context.go +++ b/api-put-object-file-context.go @@ -23,11 +23,11 @@ import ( "os" "path/filepath" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) -// FPutObjectWithContext - Create an object in a bucket, with contents from file at filePath. Allows request cancellation. -func (c Client) FPutObjectWithContext(ctx context.Context, bucketName, objectName, filePath string, opts PutObjectOptions) (n int64, err error) { +// FPutObject - Create an object in a bucket, with contents from file at filePath. Allows request cancellation. +func (c Client) FPutObject(ctx context.Context, bucketName, objectName, filePath string, opts PutObjectOptions) (n int64, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return 0, err @@ -60,5 +60,5 @@ func (c Client) FPutObjectWithContext(ctx context.Context, bucketName, objectNam opts.ContentType = "application/octet-stream" } } - return c.PutObjectWithContext(ctx, bucketName, objectName, fileReader, fileSize, opts) + return c.PutObject(ctx, bucketName, objectName, fileReader, fileSize, opts) } diff --git a/api-put-object-file.go b/api-put-object-file.go deleted file mode 100644 index 23df6cd1c..000000000 --- a/api-put-object-file.go +++ /dev/null @@ -1,27 +0,0 @@ -/* - * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "context" -) - -// FPutObject - Create an object in a bucket, with contents from file at filePath -func (c Client) FPutObject(bucketName, objectName, filePath string, opts PutObjectOptions) (n int64, err error) { - return c.FPutObjectWithContext(context.Background(), bucketName, objectName, filePath, opts) -} diff --git a/api-put-object-multipart.go b/api-put-object-multipart.go index 62af16cc2..c80cdae68 100644 --- a/api-put-object-multipart.go +++ b/api-put-object-multipart.go @@ -32,8 +32,8 @@ import ( "strconv" "strings" - "github.com/minio/minio-go/v6/pkg/encrypt" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/encrypt" + "github.com/minio/minio-go/v7/pkg/s3utils" ) func (c Client) putObjectMultipart(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, diff --git a/api-put-object-streaming.go b/api-put-object-streaming.go index f0c5b1331..2fedc6584 100644 --- a/api-put-object-streaming.go +++ b/api-put-object-streaming.go @@ -27,7 +27,7 @@ import ( "sort" "strings" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) // putObjectMultipartStream - upload a large object using diff --git a/api-put-object.go b/api-put-object.go index dbb7c8d53..5518758b7 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -28,8 +28,8 @@ import ( "sort" "time" - "github.com/minio/minio-go/v6/pkg/encrypt" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/encrypt" + "github.com/minio/minio-go/v7/pkg/s3utils" "golang.org/x/net/http/httpguts" ) @@ -168,13 +168,18 @@ func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].Part // - For size input as -1 PutObject does a multipart Put operation // until input stream reaches EOF. Maximum object size that can // be uploaded through this operation will be 5TiB. -func (c Client) PutObject(bucketName, objectName string, reader io.Reader, objectSize int64, +func (c Client) PutObject(ctx context.Context, bucketName, objectName string, reader io.Reader, objectSize int64, opts PutObjectOptions) (n int64, err error) { if objectSize < 0 && opts.DisableMultipart { return 0, errors.New("object size must be provided with disable multipart upload") } - return c.PutObjectWithContext(context.Background(), bucketName, objectName, reader, objectSize, opts) + err = opts.validate() + if err != nil { + return 0, err + } + + return c.putObjectCommon(ctx, bucketName, objectName, reader, objectSize, opts) } func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) { diff --git a/api-remove.go b/api-remove.go index 4160b0d3a..415cefb03 100644 --- a/api-remove.go +++ b/api-remove.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,19 +25,14 @@ import ( "net/http" "net/url" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) -// RemoveBucket is the wrapper for RemoveBucketWithContext -func (c Client) RemoveBucket(bucketName string) error { - return c.RemoveBucketWithContext(context.Background(), bucketName) -} - -// RemoveBucketWithContext deletes the bucket name. +// RemoveBucket deletes the bucket name. // // All objects (including all object versions and delete markers). // in the bucket must be deleted before successfully attempting this request. -func (c Client) RemoveBucketWithContext(ctx context.Context, bucketName string) error { +func (c Client) RemoveBucket(ctx context.Context, bucketName string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -64,8 +59,8 @@ func (c Client) RemoveBucketWithContext(ctx context.Context, bucketName string) } // RemoveObject remove an object from a bucket. -func (c Client) RemoveObject(bucketName, objectName string) error { - return c.RemoveObjectWithOptions(bucketName, objectName, RemoveObjectOptions{}) +func (c Client) RemoveObject(ctx context.Context, bucketName, objectName string) error { + return c.RemoveObjectWithOptions(ctx, bucketName, objectName, RemoveObjectOptions{}) } // RemoveObjectOptions represents options specified by user for RemoveObject call @@ -75,7 +70,7 @@ type RemoveObjectOptions struct { } // RemoveObjectWithOptions removes an object from a bucket. -func (c Client) RemoveObjectWithOptions(bucketName, objectName string, opts RemoveObjectOptions) error { +func (c Client) RemoveObjectWithOptions(ctx context.Context, bucketName, objectName string, opts RemoveObjectOptions) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -100,7 +95,7 @@ func (c Client) RemoveObjectWithOptions(bucketName, objectName string, opts Remo headers.Set(amzBypassGovernance, "true") } // Execute DELETE on objectName. - resp, err := c.executeMethod(context.Background(), "DELETE", requestMetadata{ + resp, err := c.executeMethod(ctx, "DELETE", requestMetadata{ bucketName: bucketName, objectName: objectName, contentSHA256Hex: emptySHA256Hex, @@ -163,8 +158,10 @@ func processRemoveMultiObjectsResponse(body io.Reader, objects []string, errorCh } } -// RemoveObjectsWithContext - Identical to RemoveObjects call, but accepts context to facilitate request cancellation. -func (c Client) RemoveObjectsWithContext(ctx context.Context, bucketName string, objectsCh <-chan string) <-chan RemoveObjectError { +// RemoveObjects removes multiple objects from a bucket. +// The list of objects to remove are received from objectsCh. +// Remove failures are sent back via error channel. +func (c Client) RemoveObjects(ctx context.Context, bucketName string, objectsCh <-chan string) <-chan RemoveObjectError { errorCh := make(chan RemoveObjectError, 1) // Validate if bucket name is valid. @@ -261,13 +258,6 @@ func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh } } -// RemoveObjects removes multiple objects from a bucket. -// The list of objects to remove are received from objectsCh. -// Remove failures are sent back via error channel. -func (c Client) RemoveObjects(bucketName string, objectsCh <-chan string) <-chan RemoveObjectError { - return c.RemoveObjectsWithContext(context.Background(), bucketName, objectsCh) -} - // RemoveObjectsOptions represents options specified by user for RemoveObjects call type RemoveObjectsOptions struct { GovernanceBypass bool @@ -276,13 +266,7 @@ type RemoveObjectsOptions struct { // RemoveObjectsWithOptions removes multiple objects from a bucket. // The list of objects to remove are received from objectsCh. // Remove failures are sent back via error channel. -func (c Client) RemoveObjectsWithOptions(bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError { - return c.RemoveObjectsWithOptionsContext(context.Background(), bucketName, objectsCh, opts) -} - -// RemoveObjectsWithOptionsContext - Identical to RemoveObjects call, but accepts context to -// facilitate request cancellation and options to bypass governance retention -func (c Client) RemoveObjectsWithOptionsContext(ctx context.Context, bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError { +func (c Client) RemoveObjectsWithOptions(ctx context.Context, bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError { errorCh := make(chan RemoveObjectError, 1) // Validate if bucket name is valid. @@ -306,13 +290,8 @@ func (c Client) RemoveObjectsWithOptionsContext(ctx context.Context, bucketName return errorCh } -// RemoveIncompleteUpload is a wrapper for RemoveIncompleteUploadWithContext -func (c Client) RemoveIncompleteUpload(bucketName, objectName string) error { - return c.RemoveIncompleteUploadWithContext(context.Background(), bucketName, objectName) -} - -// RemoveIncompleteUploadWithContext aborts an partially uploaded object. -func (c Client) RemoveIncompleteUploadWithContext(ctx context.Context, bucketName, objectName string) error { +// RemoveIncompleteUpload aborts an partially uploaded object. +func (c Client) RemoveIncompleteUpload(ctx context.Context, bucketName, objectName string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err diff --git a/api-select.go b/api-select.go index f9f4dda58..be74fa353 100644 --- a/api-select.go +++ b/api-select.go @@ -31,8 +31,8 @@ import ( "net/url" "strings" - "github.com/minio/minio-go/v6/pkg/encrypt" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/encrypt" + "github.com/minio/minio-go/v7/pkg/s3utils" ) // CSVFileHeaderInfo - is the parameter for whether to utilize headers. diff --git a/api-stat.go b/api-stat.go index 5da981b6c..51f9ae9f3 100644 --- a/api-stat.go +++ b/api-stat.go @@ -21,17 +21,12 @@ import ( "context" "net/http" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) -// BucketExists verify if bucket exists and you have permission to access it. -func (c Client) BucketExists(bucketName string) (bool, error) { - return c.BucketExistsWithContext(context.Background(), bucketName) -} - -// BucketExistsWithContext verify if bucket exists and you have permission to access it. Allows for a Context to +// BucketExists verifies if bucket exists and you have permission to access it. Allows for a Context to // control cancellations and timeouts. -func (c Client) BucketExistsWithContext(ctx context.Context, bucketName string) (bool, error) { +func (c Client) BucketExists(ctx context.Context, bucketName string) (bool, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return false, err @@ -62,13 +57,7 @@ func (c Client) BucketExistsWithContext(ctx context.Context, bucketName string) } // StatObject verifies if object exists and you have permission to access. -func (c Client) StatObject(bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { - return c.StatObjectWithContext(context.Background(), bucketName, objectName, opts) -} - -// StatObjectWithContext verifies if object exists and you have permission to access with a context to control -// cancellations and timeouts. -func (c Client) StatObjectWithContext(ctx context.Context, bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { +func (c Client) StatObject(ctx context.Context, bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ObjectInfo{}, err diff --git a/api.go b/api.go index 81327de81..98c928cda 100644 --- a/api.go +++ b/api.go @@ -37,9 +37,9 @@ import ( "time" md5simd "github.com/minio/md5-simd" - "github.com/minio/minio-go/v6/pkg/credentials" - "github.com/minio/minio-go/v6/pkg/s3utils" - "github.com/minio/minio-go/v6/pkg/signer" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/minio-go/v7/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/signer" "golang.org/x/net/publicsuffix" ) @@ -107,7 +107,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.58" + libraryVersion = "v7.0.0" ) // User Agent should always following the below style. @@ -649,7 +649,7 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque // Instantiate a new request. var req *http.Request - req, err = c.newRequest(method, metadata) + req, err = c.newRequest(ctx, method, metadata) if err != nil { errResponse := ToErrorResponse(err) if isS3CodeRetryable(errResponse.Code) { @@ -756,7 +756,7 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque } // newRequest - instantiate a new HTTP request for a given method. -func (c Client) newRequest(method string, metadata requestMetadata) (req *http.Request, err error) { +func (c Client) newRequest(ctx context.Context, method string, metadata requestMetadata) (req *http.Request, err error) { // If no method is supplied default to 'POST'. if method == "" { method = "POST" @@ -766,7 +766,7 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R if location == "" { if metadata.bucketName != "" { // Gather location only if bucketName is present. - location, err = c.getBucketLocation(metadata.bucketName) + location, err = c.getBucketLocation(ctx, metadata.bucketName) if err != nil { return nil, err } diff --git a/api_unit_test.go b/api_unit_test.go index 2dca213d4..4dd0162d9 100644 --- a/api_unit_test.go +++ b/api_unit_test.go @@ -21,8 +21,8 @@ import ( "net/url" "testing" - "github.com/minio/minio-go/v6/pkg/credentials" - "github.com/minio/minio-go/v6/pkg/policy" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/minio-go/v7/pkg/policy" ) // Tests valid hosts for location. diff --git a/bucket-cache.go b/bucket-cache.go index 305367bb5..d60994424 100644 --- a/bucket-cache.go +++ b/bucket-cache.go @@ -18,15 +18,16 @@ package minio import ( + "context" "net" "net/http" "net/url" "path" "sync" - "github.com/minio/minio-go/v6/pkg/credentials" - "github.com/minio/minio-go/v6/pkg/s3utils" - "github.com/minio/minio-go/v6/pkg/signer" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/minio-go/v7/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/signer" ) // bucketLocationCache - Provides simple mechanism to hold bucket @@ -72,16 +73,16 @@ func (r *bucketLocationCache) Delete(bucketName string) { // GetBucketLocation - get location for the bucket name from location cache, if not // fetch freshly by making a new request. -func (c Client) GetBucketLocation(bucketName string) (string, error) { +func (c Client) GetBucketLocation(ctx context.Context, bucketName string) (string, error) { if err := s3utils.CheckValidBucketName(bucketName); err != nil { return "", err } - return c.getBucketLocation(bucketName) + return c.getBucketLocation(ctx, bucketName) } // getBucketLocation - Get location for the bucketName from location map cache, if not // fetch freshly by making a new request. -func (c Client) getBucketLocation(bucketName string) (string, error) { +func (c Client) getBucketLocation(ctx context.Context, bucketName string) (string, error) { if err := s3utils.CheckValidBucketName(bucketName); err != nil { return "", err } diff --git a/bucket-cache_test.go b/bucket-cache_test.go index 3c14a8b99..44c13b115 100644 --- a/bucket-cache_test.go +++ b/bucket-cache_test.go @@ -27,8 +27,8 @@ import ( "reflect" "testing" - "github.com/minio/minio-go/v6/pkg/credentials" - "github.com/minio/minio-go/v6/pkg/signer" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/minio-go/v7/pkg/signer" ) // Test validates `newBucketLocationCache`. diff --git a/bucket-notification.go b/bucket-notification.go index 7503b01b9..7a7d578f0 100644 --- a/bucket-notification.go +++ b/bucket-notification.go @@ -22,7 +22,7 @@ import ( "errors" "fmt" - "github.com/minio/minio-go/v6/pkg/set" + "github.com/minio/minio-go/v7/pkg/set" ) // NotificationEventType is a S3 notification event associated to the bucket notification configuration diff --git a/core.go b/core.go index b54b8d379..605e7ac5f 100644 --- a/core.go +++ b/core.go @@ -22,7 +22,7 @@ import ( "io" "net/http" - "github.com/minio/minio-go/v6/pkg/encrypt" + "github.com/minio/minio-go/v7/pkg/encrypt" ) // Core - Inherits Client and adds new methods to expose the low level S3 APIs. @@ -55,63 +55,39 @@ func (c Core) ListObjectsV2(bucketName, objectPrefix, continuationToken string, return c.listObjectsV2Query(context.Background(), bucketName, objectPrefix, continuationToken, fetchOwner, false, delimiter, maxkeys, startAfter) } -// CopyObjectWithContext - copies an object from source object to destination object on server side. -func (c Core) CopyObjectWithContext(ctx context.Context, sourceBucket, sourceObject, destBucket, destObject string, metadata map[string]string) (ObjectInfo, error) { - return c.copyObjectDo(ctx, sourceBucket, sourceObject, destBucket, destObject, metadata) -} - // CopyObject - copies an object from source object to destination object on server side. -func (c Core) CopyObject(sourceBucket, sourceObject, destBucket, destObject string, metadata map[string]string) (ObjectInfo, error) { - return c.CopyObjectWithContext(context.Background(), sourceBucket, sourceObject, destBucket, destObject, metadata) -} - -// CopyObjectPartWithContext - creates a part in a multipart upload by copying (a -// part of) an existing object. -func (c Core) CopyObjectPartWithContext(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, - partID int, startOffset, length int64, metadata map[string]string) (p CompletePart, err error) { - - return c.copyObjectPartDo(ctx, srcBucket, srcObject, destBucket, destObject, uploadID, - partID, startOffset, length, metadata) +func (c Core) CopyObject(ctx context.Context, sourceBucket, sourceObject, destBucket, destObject string, metadata map[string]string) (ObjectInfo, error) { + return c.copyObjectDo(ctx, sourceBucket, sourceObject, destBucket, destObject, metadata) } // CopyObjectPart - creates a part in a multipart upload by copying (a // part of) an existing object. -func (c Core) CopyObjectPart(srcBucket, srcObject, destBucket, destObject string, uploadID string, +func (c Core) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset, length int64, metadata map[string]string) (p CompletePart, err error) { - return c.CopyObjectPartWithContext(context.Background(), srcBucket, srcObject, destBucket, destObject, uploadID, + return c.copyObjectPartDo(ctx, srcBucket, srcObject, destBucket, destObject, uploadID, partID, startOffset, length, metadata) } -// PutObjectWithContext - Upload object. Uploads using single PUT call. -func (c Core) PutObjectWithContext(ctx context.Context, bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, opts PutObjectOptions) (ObjectInfo, error) { - return c.putObjectDo(ctx, bucket, object, data, md5Base64, sha256Hex, size, opts) -} - // PutObject - Upload object. Uploads using single PUT call. -func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, opts PutObjectOptions) (ObjectInfo, error) { - return c.PutObjectWithContext(context.Background(), bucket, object, data, size, md5Base64, sha256Hex, opts) +func (c Core) PutObject(ctx context.Context, bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, opts PutObjectOptions) (ObjectInfo, error) { + return c.putObjectDo(ctx, bucket, object, data, md5Base64, sha256Hex, size, opts) } // NewMultipartUpload - Initiates new multipart upload and returns the new uploadID. -func (c Core) NewMultipartUpload(bucket, object string, opts PutObjectOptions) (uploadID string, err error) { - result, err := c.initiateMultipartUpload(context.Background(), bucket, object, opts) +func (c Core) NewMultipartUpload(ctx context.Context, bucket, object string, opts PutObjectOptions) (uploadID string, err error) { + result, err := c.initiateMultipartUpload(ctx, bucket, object, opts) return result.UploadID, err } // ListMultipartUploads - List incomplete uploads. -func (c Core) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartUploadsResult, err error) { - return c.listMultipartUploadsQuery(context.Background(), bucket, keyMarker, uploadIDMarker, prefix, delimiter, maxUploads) -} - -// PutObjectPartWithContext - Upload an object part. -func (c Core) PutObjectPartWithContext(ctx context.Context, bucket, object, uploadID string, partID int, data io.Reader, size int64, md5Base64, sha256Hex string, sse encrypt.ServerSide) (ObjectPart, error) { - return c.uploadPart(ctx, bucket, object, uploadID, data, partID, md5Base64, sha256Hex, size, sse) +func (c Core) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartUploadsResult, err error) { + return c.listMultipartUploadsQuery(ctx, bucket, keyMarker, uploadIDMarker, prefix, delimiter, maxUploads) } // PutObjectPart - Upload an object part. -func (c Core) PutObjectPart(bucket, object, uploadID string, partID int, data io.Reader, size int64, md5Base64, sha256Hex string, sse encrypt.ServerSide) (ObjectPart, error) { - return c.PutObjectPartWithContext(context.Background(), bucket, object, uploadID, partID, data, size, md5Base64, sha256Hex, sse) +func (c Core) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data io.Reader, size int64, md5Base64, sha256Hex string, sse encrypt.ServerSide) (ObjectPart, error) { + return c.uploadPart(ctx, bucket, object, uploadID, data, partID, md5Base64, sha256Hex, size, sse) } // ListObjectParts - List uploaded parts of an incomplete upload.x @@ -119,67 +95,38 @@ func (c Core) ListObjectParts(ctx context.Context, bucket, object, uploadID stri return c.listObjectPartsQuery(ctx, bucket, object, uploadID, partNumberMarker, maxParts) } -// CompleteMultipartUploadWithContext - Concatenate uploaded parts and commit to an object. -func (c Core) CompleteMultipartUploadWithContext(ctx context.Context, bucket, object, uploadID string, parts []CompletePart) (string, error) { +// CompleteMultipartUpload - Concatenate uploaded parts and commit to an object. +func (c Core) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, parts []CompletePart) (string, error) { res, err := c.completeMultipartUpload(ctx, bucket, object, uploadID, completeMultipartUpload{ Parts: parts, }) return res.ETag, err } -// CompleteMultipartUpload - Concatenate uploaded parts and commit to an object. -func (c Core) CompleteMultipartUpload(bucket, object, uploadID string, parts []CompletePart) (string, error) { - return c.CompleteMultipartUploadWithContext(context.Background(), bucket, object, uploadID, parts) -} - -// AbortMultipartUploadWithContext - Abort an incomplete upload. -func (c Core) AbortMultipartUploadWithContext(ctx context.Context, bucket, object, uploadID string) error { - return c.abortMultipartUpload(ctx, bucket, object, uploadID) -} - // AbortMultipartUpload - Abort an incomplete upload. -func (c Core) AbortMultipartUpload(bucket, object, uploadID string) error { - return c.AbortMultipartUploadWithContext(context.Background(), bucket, object, uploadID) +func (c Core) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error { + return c.abortMultipartUpload(ctx, bucket, object, uploadID) } // GetBucketPolicy - fetches bucket access policy for a given bucket. -func (c Core) GetBucketPolicy(bucket string) (string, error) { - return c.getBucketPolicy(context.Background(), bucket) +func (c Core) GetBucketPolicy(ctx context.Context, bucket string) (string, error) { + return c.getBucketPolicy(ctx, bucket) } // PutBucketPolicy - applies a new bucket access policy for a given bucket. -func (c Core) PutBucketPolicy(bucket, bucketPolicy string) error { - return c.PutBucketPolicyWithContext(context.Background(), bucket, bucketPolicy) -} - -// PutBucketPolicyWithContext - applies a new bucket access policy for a given bucket with a context to control -// cancellations and timeouts. -func (c Core) PutBucketPolicyWithContext(ctx context.Context, bucket, bucketPolicy string) error { +func (c Core) PutBucketPolicy(ctx context.Context, bucket, bucketPolicy string) error { return c.putBucketPolicy(ctx, bucket, bucketPolicy) } -// GetObjectWithContext is a lower level API implemented to support reading -// partial objects and also downloading objects with special conditions -// matching etag, modtime etc. -func (c Core) GetObjectWithContext(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, http.Header, error) { - return c.getObject(ctx, bucketName, objectName, opts) -} - // GetObject is a lower level API implemented to support reading // partial objects and also downloading objects with special conditions // matching etag, modtime etc. -func (c Core) GetObject(bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, http.Header, error) { - return c.GetObjectWithContext(context.Background(), bucketName, objectName, opts) -} - -// StatObjectWithContext is a lower level API implemented to support special -// conditions matching etag, modtime on a request. -func (c Core) StatObjectWithContext(ctx context.Context, bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { - return c.statObject(ctx, bucketName, objectName, opts) +func (c Core) GetObject(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, http.Header, error) { + return c.getObject(ctx, bucketName, objectName, opts) } // StatObject is a lower level API implemented to support special // conditions matching etag, modtime on a request. -func (c Core) StatObject(bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { - return c.StatObjectWithContext(context.Background(), bucketName, objectName, opts) +func (c Core) StatObject(ctx context.Context, bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { + return c.statObject(ctx, bucketName, objectName, opts) } diff --git a/core_test.go b/core_test.go index ada24605c..4cc52f669 100644 --- a/core_test.go +++ b/core_test.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 MinIO, Inc. + * Copyright 2017-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ package minio import ( "bytes" + "context" "io" "log" "net/http" @@ -92,7 +93,7 @@ func TestGetObjectCore(t *testing.T) { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { t.Fatal("Error:", err, bucketName) } @@ -102,7 +103,7 @@ func TestGetObjectCore(t *testing.T) { // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.Client.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), PutObjectOptions{ + n, err := c.Client.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), PutObjectOptions{ ContentType: "binary/octet-stream", }) if err != nil { @@ -123,7 +124,7 @@ func TestGetObjectCore(t *testing.T) { opts := GetObjectOptions{} opts.SetRange(offset, offset+int64(len(buf1))-1) - reader, objectInfo, _, err := c.GetObject(bucketName, objectName, opts) + reader, objectInfo, _, err := c.GetObject(context.Background(), bucketName, objectName, opts) if err != nil { t.Fatal(err) } @@ -142,7 +143,7 @@ func TestGetObjectCore(t *testing.T) { offset += 512 opts.SetRange(offset, offset+int64(len(buf2))-1) - reader, objectInfo, _, err = c.GetObject(bucketName, objectName, opts) + reader, objectInfo, _, err = c.GetObject(context.Background(), bucketName, objectName, opts) if err != nil { t.Fatal(err) } @@ -161,7 +162,7 @@ func TestGetObjectCore(t *testing.T) { } opts.SetRange(0, int64(len(buf3))) - reader, objectInfo, _, err = c.GetObject(bucketName, objectName, opts) + reader, objectInfo, _, err = c.GetObject(context.Background(), bucketName, objectName, opts) if err != nil { t.Fatal(err) } @@ -182,7 +183,7 @@ func TestGetObjectCore(t *testing.T) { opts = GetObjectOptions{} opts.SetMatchETag("etag") - _, _, _, err = c.GetObject(bucketName, objectName, opts) + _, _, _, err = c.GetObject(context.Background(), bucketName, objectName, opts) if err == nil { t.Fatal("Unexpected GetObject should fail with mismatching etags") } @@ -192,7 +193,7 @@ func TestGetObjectCore(t *testing.T) { opts = GetObjectOptions{} opts.SetMatchETagExcept("etag") - reader, objectInfo, _, err = c.GetObject(bucketName, objectName, opts) + reader, objectInfo, _, err = c.GetObject(context.Background(), bucketName, objectName, opts) if err != nil { t.Fatal(err) } @@ -212,7 +213,7 @@ func TestGetObjectCore(t *testing.T) { opts = GetObjectOptions{} opts.SetRange(0, 0) - reader, objectInfo, _, err = c.GetObject(bucketName, objectName, opts) + reader, objectInfo, _, err = c.GetObject(context.Background(), bucketName, objectName, opts) if err != nil { t.Fatal(err) } @@ -231,7 +232,7 @@ func TestGetObjectCore(t *testing.T) { opts.SetRange(offset, offset+int64(len(buf2))-1) contentLength := len(buf2) var header http.Header - _, _, header, err = c.GetObject(bucketName, objectName, opts) + _, _, header, err = c.GetObject(context.Background(), bucketName, objectName, opts) if err != nil { t.Fatal(err) } @@ -244,11 +245,11 @@ func TestGetObjectCore(t *testing.T) { t.Fatalf("Error: Content Length in response header %v, not equal to set content length %v\n", contentLengthValue, contentLength) } - err = c.RemoveObject(bucketName, objectName) + err = c.RemoveObject(context.Background(), bucketName, objectName) if err != nil { t.Fatal("Error: ", err) } - err = c.RemoveBucket(bucketName) + err = c.RemoveBucket(context.Background(), bucketName) if err != nil { t.Fatal("Error:", err) } @@ -285,7 +286,7 @@ func TestGetObjectContentEncoding(t *testing.T) { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { t.Fatal("Error:", err, bucketName) } @@ -295,7 +296,7 @@ func TestGetObjectContentEncoding(t *testing.T) { // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.Client.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), PutObjectOptions{ + n, err := c.Client.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), PutObjectOptions{ ContentEncoding: "gzip", }) if err != nil { @@ -306,7 +307,7 @@ func TestGetObjectContentEncoding(t *testing.T) { t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) } - rwc, objInfo, _, err := c.GetObject(bucketName, objectName, GetObjectOptions{}) + rwc, objInfo, _, err := c.GetObject(context.Background(), bucketName, objectName, GetObjectOptions{}) if err != nil { t.Fatalf("Error: %v", err) } @@ -322,11 +323,11 @@ func TestGetObjectContentEncoding(t *testing.T) { t.Fatalf("Unexpected content-encoding found, want gzip, got %v", value) } - err = c.RemoveObject(bucketName, objectName) + err = c.RemoveObject(context.Background(), bucketName, objectName) if err != nil { t.Fatal("Error: ", err) } - err = c.RemoveBucket(bucketName) + err = c.RemoveBucket(context.Background(), bucketName) if err != nil { t.Fatal("Error:", err) } @@ -362,14 +363,14 @@ func TestGetBucketPolicy(t *testing.T) { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { t.Fatal("Error:", err, bucketName) } // Verify if bucket exits and you have access. var exists bool - exists, err = c.BucketExists(bucketName) + exists, err = c.BucketExists(context.Background(), bucketName) if err != nil { t.Fatal("Error:", err, bucketName) } @@ -378,7 +379,7 @@ func TestGetBucketPolicy(t *testing.T) { } // Asserting the default bucket policy. - bucketPolicy, err := c.GetBucketPolicy(bucketName) + bucketPolicy, err := c.GetBucketPolicy(context.Background(), bucketName) if err != nil { errResp := ToErrorResponse(err) if errResp.Code != "NoSuchBucketPolicy" { @@ -389,7 +390,7 @@ func TestGetBucketPolicy(t *testing.T) { t.Errorf("Bucket policy expected %#v, got %#v", "", bucketPolicy) } - err = c.RemoveBucket(bucketName) + err = c.RemoveBucket(context.Background(), bucketName) if err != nil { t.Fatal("Error:", err) } @@ -425,7 +426,7 @@ func TestCoreCopyObject(t *testing.T) { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { t.Fatal("Error:", err, bucketName) } @@ -440,7 +441,7 @@ func TestCoreCopyObject(t *testing.T) { "Content-Type": "binary/octet-stream", }, } - objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", putopts) + objInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", putopts) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } @@ -452,7 +453,7 @@ func TestCoreCopyObject(t *testing.T) { destBucketName := bucketName destObjectName := objectName + "-dest" - cobjInfo, err := c.CopyObject(bucketName, objectName, destBucketName, destObjectName, map[string]string{ + cobjInfo, err := c.CopyObject(context.Background(), bucketName, objectName, destBucketName, destObjectName, map[string]string{ "X-Amz-Metadata-Directive": "REPLACE", "Content-Type": "application/javascript", }) @@ -464,7 +465,7 @@ func TestCoreCopyObject(t *testing.T) { } // Attempt to read from destBucketName and object name. - r, err := c.Client.GetObject(destBucketName, destObjectName, GetObjectOptions{}) + r, err := c.Client.GetObject(context.Background(), destBucketName, destObjectName, GetObjectOptions{}) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } @@ -495,17 +496,17 @@ func TestCoreCopyObject(t *testing.T) { t.Fatal("Error: object is already closed, should return error") } - err = c.RemoveObject(bucketName, objectName) + err = c.RemoveObject(context.Background(), bucketName, objectName) if err != nil { t.Fatal("Error: ", err) } - err = c.RemoveObject(destBucketName, destObjectName) + err = c.RemoveObject(context.Background(), destBucketName, destObjectName) if err != nil { t.Fatal("Error: ", err) } - err = c.RemoveBucket(bucketName) + err = c.RemoveBucket(context.Background(), bucketName) if err != nil { t.Fatal("Error:", err) } @@ -543,7 +544,7 @@ func TestCoreCopyObjectPart(t *testing.T) { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { t.Fatal("Error:", err, bucketName) } @@ -558,7 +559,7 @@ func TestCoreCopyObjectPart(t *testing.T) { } // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", putopts) + objInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", putopts) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } @@ -570,7 +571,7 @@ func TestCoreCopyObjectPart(t *testing.T) { destBucketName := bucketName destObjectName := objectName + "-dest" - uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, PutObjectOptions{}) + uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, PutObjectOptions{}) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } @@ -580,31 +581,31 @@ func TestCoreCopyObjectPart(t *testing.T) { // `objectName`. // First of three parts - fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, nil) + fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, nil) if err != nil { t.Fatal("Error:", err, destBucketName, destObjectName) } // Second of three parts - sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, nil) + sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, nil) if err != nil { t.Fatal("Error:", err, destBucketName, destObjectName) } // Last of three parts - lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, nil) + lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, nil) if err != nil { t.Fatal("Error:", err, destBucketName, destObjectName) } // Complete the multipart upload - _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []CompletePart{fstPart, sndPart, lstPart}) + _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []CompletePart{fstPart, sndPart, lstPart}) if err != nil { t.Fatal("Error:", err, destBucketName, destObjectName) } // Stat the object and check its length matches - objInfo, err = c.StatObject(destBucketName, destObjectName, StatObjectOptions{}) + objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, StatObjectOptions{}) if err != nil { t.Fatal("Error:", err, destBucketName, destObjectName) } @@ -616,7 +617,7 @@ func TestCoreCopyObjectPart(t *testing.T) { // Now we read the data back getOpts := GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) - r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { t.Fatal("Error:", err, destBucketName, destObjectName) } @@ -630,7 +631,7 @@ func TestCoreCopyObjectPart(t *testing.T) { } getOpts.SetRange(5*1024*1024, 0) - r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { t.Fatal("Error:", err, destBucketName, destObjectName) } @@ -646,15 +647,15 @@ func TestCoreCopyObjectPart(t *testing.T) { t.Fatal("Got unexpected data in last byte of copied object!") } - if err := c.RemoveObject(destBucketName, destObjectName); err != nil { + if err := c.RemoveObject(context.Background(), destBucketName, destObjectName); err != nil { t.Fatal("Error: ", err) } - if err := c.RemoveObject(bucketName, objectName); err != nil { + if err := c.RemoveObject(context.Background(), bucketName, objectName); err != nil { t.Fatal("Error: ", err) } - if err := c.RemoveBucket(bucketName); err != nil { + if err := c.RemoveBucket(context.Background(), bucketName); err != nil { t.Fatal("Error: ", err) } @@ -692,7 +693,7 @@ func TestCorePutObject(t *testing.T) { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { t.Fatal("Error:", err, bucketName) } @@ -708,12 +709,12 @@ func TestCorePutObject(t *testing.T) { putopts := PutObjectOptions{ UserMetadata: metadata, } - objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "1B2M2Y8AsgTpgAmY7PhCfg==", "", putopts) + objInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "1B2M2Y8AsgTpgAmY7PhCfg==", "", putopts) if err == nil { t.Fatal("Error expected: error, got: nil(success)") } - objInfo, err = c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", putopts) + objInfo, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", putopts) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } @@ -723,7 +724,7 @@ func TestCorePutObject(t *testing.T) { } // Read the data back - r, err := c.Client.GetObject(bucketName, objectName, GetObjectOptions{}) + r, err := c.Client.GetObject(context.Background(), bucketName, objectName, GetObjectOptions{}) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } @@ -750,12 +751,12 @@ func TestCorePutObject(t *testing.T) { t.Fatal("Error: object is already closed, should return error") } - err = c.RemoveObject(bucketName, objectName) + err = c.RemoveObject(context.Background(), bucketName, objectName) if err != nil { t.Fatal("Error: ", err) } - err = c.RemoveBucket(bucketName) + err = c.RemoveBucket(context.Background(), bucketName) if err != nil { t.Fatal("Error:", err) } @@ -779,7 +780,7 @@ func TestCoreGetObjectMetadata(t *testing.T) { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = core.MakeBucket(bucketName, "us-east-1") + err = core.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { t.Fatal("Error:", err, bucketName) } @@ -791,13 +792,13 @@ func TestCoreGetObjectMetadata(t *testing.T) { UserMetadata: metadata, } - _, err = core.PutObject(bucketName, "my-objectname", + _, err = core.PutObject(context.Background(), bucketName, "my-objectname", bytes.NewReader([]byte("hello")), 5, "", "", putopts) if err != nil { log.Fatalln(err) } - reader, objInfo, _, err := core.GetObject(bucketName, "my-objectname", GetObjectOptions{}) + reader, objInfo, _, err := core.GetObject(context.Background(), bucketName, "my-objectname", GetObjectOptions{}) if err != nil { log.Fatalln(err) } @@ -807,11 +808,11 @@ func TestCoreGetObjectMetadata(t *testing.T) { log.Fatalln("Expected metadata to be available but wasn't") } - err = core.RemoveObject(bucketName, "my-objectname") + err = core.RemoveObject(context.Background(), bucketName, "my-objectname") if err != nil { t.Fatal("Error: ", err) } - err = core.RemoveBucket(bucketName) + err = core.RemoveBucket(context.Background(), bucketName) if err != nil { t.Fatal("Error:", err) } diff --git a/docs/API.md b/docs/API.md index 33f1c3244..efe2aa4b8 100644 --- a/docs/API.md +++ b/docs/API.md @@ -10,7 +10,7 @@ package main import ( "fmt" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -34,7 +34,7 @@ package main import ( "fmt" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -56,33 +56,25 @@ func main() { | [`MakeBucketWithObjectLock`](#MakeBucketWithObjectLock) | [`PutObject`](#PutObject) | [`PutObject`](#PutObject) | [`PresignedPutObject`](#PresignedPutObject) | [`GetBucketPolicy`](#GetBucketPolicy) | [`SetCustomTransport`](#SetCustomTransport) | | [`ListBuckets`](#ListBuckets) | [`CopyObject`](#CopyObject) | [`CopyObject`](#CopyObject) | [`PresignedPostPolicy`](#PresignedPostPolicy) | [`SetBucketNotification`](#SetBucketNotification) | [`TraceOn`](#TraceOn) | | [`BucketExists`](#BucketExists) | [`StatObject`](#StatObject) | [`StatObject`](#StatObject) | | [`GetBucketNotification`](#GetBucketNotification) | [`TraceOff`](#TraceOff) | -| [`RemoveBucket`](#RemoveBucket) | [`RemoveObject`](#RemoveObject) | | | [`RemoveAllBucketNotification`](#RemoveAllBucketNotification) | [`SetS3TransferAccelerate`](#SetS3TransferAccelerate) | -| [`ListObjects`](#ListObjects) | [`RemoveObjects`](#RemoveObjects) | | | [`ListenBucketNotification`](#ListenBucketNotification) | | -| [`ListObjectsV2`](#ListObjectsV2) | [`RemoveIncompleteUpload`](#RemoveIncompleteUpload) | | | [`SetBucketLifecycle`](#SetBucketLifecycle) | | -| [`ListIncompleteUploads`](#ListIncompleteUploads) | [`FPutObject`](#FPutObject) | [`FPutObject`](#FPutObject) | | [`GetBucketLifecycle`](#GetBucketLifecycle) | | -| [`SetBucketTagging`](#SetBucketTagging) | [`FGetObject`](#FGetObject) | [`FGetObject`](#FGetObject) | | [`SetObjectLockConfig`](#SetObjectLockConfig) | | -| [`GetBucketTagging`](#GetBucketTagging) | [`ComposeObject`](#ComposeObject) | [`ComposeObject`](#ComposeObject) | | [`GetObjectLockConfig`](#GetObjectLockConfig) | | -| [`DeleteBucketTagging`](#DeleteBucketTagging) | [`NewSourceInfo`](#NewSourceInfo) | [`NewSourceInfo`](#NewSourceInfo) | | [`EnableVersioning`](#EnableVersioning) | | -| | [`NewDestinationInfo`](#NewDestinationInfo) | [`NewDestinationInfo`](#NewDestinationInfo) | | [`DisableVersioning`](#DisableVersioning) | | -| | [`PutObjectWithContext`](#PutObjectWithContext) | [`PutObjectWithContext`](#PutObjectWithContext) | | [`GetBucketVersioning`](#GetBucketVersioning) | | -| | [`GetObjectWithContext`](#GetObjectWithContext) | [`GetObjectWithContext`](#GetObjectWithContext) | | [`SetBucketEncryption`](#SetBucketEncryption) | | -| | [`FPutObjectWithContext`](#FPutObjectWithContext) | [`FPutObjectWithContext`](#FPutObjectWithContext) | | [`GetBucketEncryption`](#GetBucketEncryption) | | -| | [`FGetObjectWithContext`](#FGetObjectWithContext) | [`FGetObjectWithContext`](#FGetObjectWithContext) | | [`DeleteBucketEncryption`](#DeleteBucketEncryption) | | -| | [`RemoveObjectsWithContext`](#RemoveObjectsWithContext) | | | | | -| | [`RemoveObjectsWithOptions`](#RemoveObjectsWithOptions) | | | | | -| | [`RemoveObjectsWithOptionsContext`](#RemoveObjectsWithOptionsContext) | | | | | -| | [`RemoveObjectWithOptions`](#RemoveObjectWithOptions) | | | | | -| | [`PutObjectRetention`](#PutObjectRetention) | | | | | -| | [`GetObjectRetention`](#GetObjectRetention) | | | | | +| [`RemoveBucket`](#RemoveBucket) | [`RemoveObject`](#RemoveObject) | [`FPutObject`](#FPutObject) | | [`RemoveAllBucketNotification`](#RemoveAllBucketNotification) | [`SetS3TransferAccelerate`](#SetS3TransferAccelerate) | +| [`ListObjects`](#ListObjects) | [`RemoveObjects`](#RemoveObjects) | [`FGetObject`](#FGetObject) | | [`ListenBucketNotification`](#ListenBucketNotification) | | +| [`ListObjectsV2`](#ListObjectsV2) | [`RemoveIncompleteUpload`](#RemoveIncompleteUpload) | [`ComposeObject`](#ComposeObjecet) | | [`SetBucketLifecycle`](#SetBucketLifecycle) | | +| [`ListIncompleteUploads`](#ListIncompleteUploads) | [`FPutObject`](#FPutObject) | [`NewSourceInfo`](#NewSourceInfo) | | [`GetBucketLifecycle`](#GetBucketLifecycle) | | +| [`SetBucketTagging`](#SetBucketTagging) | [`FGetObject`](#FGetObject) | [`NewDestinationInfo`](#NewDestinationInfo) | | [`SetObjectLockConfig`](#SetObjectLockConfig) | | +| [`GetBucketTagging`](#GetBucketTagging) | [`ComposeObject`](#ComposeObject) | | | [`GetObjectLockConfig`](#GetObjectLockConfig) | | +| [`DeleteBucketTagging`](#DeleteBucketTagging) | [`NewSourceInfo`](#NewSourceInfo) | | | [`EnableVersioning`](#EnableVersioning) | | +| | [`NewDestinationInfo`](#NewDestinationInfo) | | | [`DisableVersioning`](#DisableVersioning) | | +| | [`RemoveObjectsWithOptions`](#RemoveObjectsWithOptions) | | | [`GetBucketVersioning`](#GetBucketVersioning) | | +| | [`RemoveObjectWithOptions`](#RemoveObjectWithOptions) | | | [`SetBucketEncryption`](#SetBucketEncryption) | | +| | [`PutObjectRetention`](#PutObjectRetention) | | | [`GetBucketEncryption`](#GetBucketEncryption) | | +| | [`GetObjectRetention`](#GetObjectRetention) | | | [`DeleteBucketEncryption`](#DeleteBucketEncryption) | | | | [`PutObjectLegalHold`](#PutObjectLegalHold) | | | | | | | [`GetObjectLegalHold`](#GetObjectLegalHold) | | | | | | | [`SelectObjectContent`](#SelectObjectContent) | | | | | | | [`PutObjectTagging`](#PutObjectTagging) | | | | | -| | [`PutObjectTaggingWithContext`](#PutObjectTaggingWithContext) | | | | | | | [`GetObjectTagging`](#GetObjectTagging) | | | | | -| | [`GetObjectTaggingWithContext`](#GetObjectTaggingWithContext) | | | | | | | [`RemoveObjectTagging`](#RemoveObjectTagging) | | | | | -| | [`RemoveObjectTaggingWithContext`](#RemoveObjectTaggingWithContext) | | | | | +| | | | | | | ## 1. Constructor @@ -126,13 +118,14 @@ __minio.Options__ ## 2. Bucket operations -### MakeBucket(bucketName, location string) error +### MakeBucket(ctx context.Context, bucketName, location string) error Creates a new bucket. __Parameters__ | Param | Type | Description | |---|---|---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ | Name of the bucket | | `location` | _string_ | Region where the bucket is to be created. Default value is us-east-1. Other valid values are listed below. Note: When used with minio server, use the region specified in its config file (defaults to us-east-1).| | | |us-east-1 | @@ -164,7 +157,7 @@ __Example__ ```go -err = minioClient.MakeBucket("mybucket", "us-east-1") +err = minioClient.MakeBucket(context.Background(), "mybucket", "us-east-1") if err != nil { fmt.Println(err) return @@ -173,13 +166,14 @@ fmt.Println("Successfully created mybucket.") ``` -### MakeBucketWithObjectLock(bucketName, location string) error +### MakeBucketWithObjectLock(ctx context.Background, bucketName, location string) error Creates a new bucket with object lock enabled. __Parameters__ | Param | Type | Description | |---|---|---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ | Name of the bucket | | `location` | _string_ | Region where the bucket is to be created. Default value is us-east-1. Other valid values are listed below. Note: When used with minio server, use the region specified in its config file (defaults to us-east-1).| | | |us-east-1 | @@ -197,7 +191,7 @@ __Example__ ```go -err = minioClient.MakeBucketWithObjectLock("mybucket", "us-east-1") +err = minioClient.MakeBucketWithObjectLock(context.Background(), "mybucket", "us-east-1") if err != nil { fmt.Println(err) return @@ -206,11 +200,12 @@ fmt.Println("Successfully created mybucket.") ``` -### ListBuckets() ([]BucketInfo, error) +### ListBuckets(ctx context.Context) ([]BucketInfo, error) Lists all buckets. | Param | Type | Description | |---|---|---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketList` | _[]minio.BucketInfo_ | Lists of all buckets | @@ -226,7 +221,7 @@ __Example__ ```go -buckets, err := minioClient.ListBuckets() +buckets, err := minioClient.ListBuckets(context.Background()) if err != nil { fmt.Println(err) return @@ -237,7 +232,7 @@ for _, bucket := range buckets { ``` -### BucketExists(bucketName string) (found bool, err error) +### BucketExists(ctx context.Context, bucketName string) (found bool, err error) Checks if a bucket exists. __Parameters__ @@ -245,6 +240,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | @@ -260,7 +256,7 @@ __Example__ ```go -found, err := minioClient.BucketExists("mybucket") +found, err := minioClient.BucketExists(context.Background(), "mybucket") if err != nil { fmt.Println(err) return @@ -271,7 +267,7 @@ if found { ``` -### RemoveBucket(bucketName string) error +### RemoveBucket(ctx context.Context, bucketName string) error Removes a bucket, bucket should be empty to be successfully removed. __Parameters__ @@ -279,13 +275,14 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | __Example__ ```go -err = minioClient.RemoveBucket("mybucket") +err = minioClient.RemoveBucket(context.Background(), "mybucket") if err != nil { fmt.Println(err) return @@ -293,7 +290,7 @@ if err != nil { ``` -### ListObjects(bucketName, prefix string, recursive bool, doneCh chan struct{}) <-chan ObjectInfo +### ListObjects(ctx context.Context, bucketName, prefix string, recursive bool, doneCh chan struct{}) <-chan ObjectInfo Lists objects in a bucket. __Parameters__ @@ -301,6 +298,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | |`objectPrefix` |_string_ | Prefix of objects to be listed | |`recursive` | _bool_ |`true` indicates recursive style listing and `false` indicates directory style listing delimited by '/'. | @@ -331,7 +329,7 @@ doneCh := make(chan struct{}) defer close(doneCh) isRecursive := true -objectCh := minioClient.ListObjects("mybucket", "myprefix", isRecursive, doneCh) +objectCh := minioClient.ListObjects(context.Background(), "mybucket", "myprefix", isRecursive, doneCh) for object := range objectCh { if object.Err != nil { fmt.Println(object.Err) @@ -343,7 +341,7 @@ for object := range objectCh { -### ListObjectsV2(bucketName, prefix string, recursive bool, doneCh chan struct{}) <-chan ObjectInfo +### ListObjectsV2(ctx context.Context, bucketName, prefix string, recursive bool, doneCh chan struct{}) <-chan ObjectInfo Lists objects in a bucket using the recommended listing API v2 __Parameters__ @@ -351,6 +349,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | | `objectPrefix` |_string_ | Prefix of objects to be listed | | `recursive` | _bool_ |`true` indicates recursive style listing and `false` indicates directory style listing delimited by '/'. | @@ -372,7 +371,7 @@ doneCh := make(chan struct{}) defer close(doneCh) isRecursive := true -objectCh := minioClient.ListObjectsV2("mybucket", "myprefix", isRecursive, doneCh) +objectCh := minioClient.ListObjectsV2(context.Background(), "mybucket", "myprefix", isRecursive, doneCh) for object := range objectCh { if object.Err != nil { fmt.Println(object.Err) @@ -383,7 +382,7 @@ for object := range objectCh { ``` -### ListIncompleteUploads(bucketName, prefix string, recursive bool, doneCh chan struct{}) <- chan ObjectMultipartInfo +### ListIncompleteUploads(ctx context.Context, bucketName, prefix string, recursive bool, doneCh chan struct{}) <- chan ObjectMultipartInfo Lists partially uploaded objects in a bucket. @@ -392,6 +391,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | | `prefix` |_string_ | Prefix of objects that are partially uploaded | | `recursive` | _bool_ |`true` indicates recursive style listing and `false` indicates directory style listing delimited by '/'. | @@ -423,7 +423,7 @@ doneCh := make(chan struct{}) defer close(doneCh) isRecursive := true // Recursively list everything at 'myprefix' -multiPartObjectCh := minioClient.ListIncompleteUploads("mybucket", "myprefix", isRecursive, doneCh) +multiPartObjectCh := minioClient.ListIncompleteUploads(context.Background(), "mybucket", "myprefix", isRecursive, doneCh) for multiPartObject := range multiPartObjectCh { if multiPartObject.Err != nil { fmt.Println(multiPartObject.Err) @@ -434,13 +434,14 @@ for multiPartObject := range multiPartObjectCh { ``` -### SetBucketTagging(bucketName string, tags *tags.Tags) error +### SetBucketTagging(ctx context.Context, bucketName string, tags *tags.Tags) error Sets tags to a bucket. __Parameters__ | Param | Type | Description | |:-------------|:-------------|:-------------------| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| | `bucketName` | _string_ | Name of the bucket | | `tags` | _*tags.Tags_ | Bucket tags | @@ -455,20 +456,21 @@ if err != nil { log.Fatalln(err) } -err = minioClient.SetBucketTagging("my-bucketname", tags) +err = minioClient.SetBucketTagging(context.Background(), "my-bucketname", tags) if err != nil { log.Fatalln(err) } ``` -### GetBucketTagging(bucketName string) (*tags.Tags, error) +### GetBucketTagging(ctx context.Context, bucketName string) (*tags.Tags, error) Gets tags of a bucket. __Parameters__ | Param | Type | Description | |:-------------|:-------------|:-------------------| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| | `bucketName` | _string_ | Name of the bucket | __Return Value__ @@ -479,7 +481,7 @@ __Return Value__ __Example__ ```go -tags, err := minioClient.GetBucketTagging("my-bucketname") +tags, err := minioClient.GetBucketTagging(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } @@ -488,18 +490,19 @@ fmt.Printf("Fetched Object Tags: %v\n", tags) ``` -### DeleteBucketTagging(bucketName string) error +### DeleteBucketTagging(ctx context.Context, bucketName string) error Deletes all tags of a bucket. __Parameters__ | Param | Type | Description | |:-------------|:-------------|:-------------------| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| | `bucketName` | _string_ | Name of the bucket | __Example__ ```go -err := minioClient.DeleteBucketTagging("my-bucketname") +err := minioClient.DeleteBucketTagging(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } @@ -508,7 +511,7 @@ if err != nil { ## 3. Object operations -### GetObject(bucketName, objectName string, opts GetObjectOptions) (*Object, error) +### GetObject(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (*Object, error) Returns a stream of the object data. Most of the common errors occur when reading the stream. @@ -517,6 +520,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | |`objectName` | _string_ |Name of the object | |`opts` | _minio.GetObjectOptions_ | Options for GET requests specifying additional options like encryption, If-Match | @@ -526,7 +530,7 @@ __minio.GetObjectOptions__ |Field | Type | Description | |:---|:---|:---| -| `opts.ServerSideEncryption` | _encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v6) | +| `opts.ServerSideEncryption` | _encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v7) | __Return Value__ @@ -540,7 +544,7 @@ __Example__ ```go -object, err := minioClient.GetObject("mybucket", "myobject", minio.GetObjectOptions{}) +object, err := minioClient.GetObject(context.Background(), "mybucket", "myobject", minio.GetObjectOptions{}) if err != nil { fmt.Println(err) return @@ -557,7 +561,7 @@ if _, err = io.Copy(localFile, object); err != nil { ``` -### FGetObject(bucketName, objectName, filePath string, opts GetObjectOptions) error +### FGetObject(ctx context.Context, bucketName, objectName, filePath string, opts GetObjectOptions) error Downloads and saves the object as a file in the local filesystem. __Parameters__ @@ -565,6 +569,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | |`objectName` | _string_ |Name of the object | |`filePath` | _string_ |Path to download object to | @@ -575,84 +580,7 @@ __Example__ ```go -err = minioClient.FGetObject("mybucket", "myobject", "/tmp/myobject", minio.GetObjectOptions{}) -if err != nil { - fmt.Println(err) - return -} -``` - -### GetObjectWithContext(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (*Object, error) -Identical to GetObject operation, but accepts a context for request cancellation. - -__Parameters__ - - -|Param |Type |Description | -|:---|:---| :---| -|`ctx` | _context.Context_ |Request context | -|`bucketName` | _string_ |Name of the bucket | -|`objectName` | _string_ |Name of the object | -|`opts` | _minio.GetObjectOptions_ | Options for GET requests specifying additional options like encryption, If-Match | - - -__Return Value__ - - -|Param |Type |Description | -|:---|:---| :---| -|`object` | _*minio.Object_ |_minio.Object_ represents object reader. It implements io.Reader, io.Seeker, io.ReaderAt and io.Closer interfaces. | - - -__Example__ - - -```go -ctx, cancel := context.WithTimeout(context.Background(), 100 * time.Second) -defer cancel() - -object, err := minioClient.GetObjectWithContext(ctx, "mybucket", "myobject", minio.GetObjectOptions{}) -if err != nil { - fmt.Println(err) - return -} - -localFile, err := os.Create("/tmp/local-file.jpg") -if err != nil { - fmt.Println(err) - return -} - -if _, err = io.Copy(localFile, object); err != nil { - fmt.Println(err) - return -} -``` - - -### FGetObjectWithContext(ctx context.Context, bucketName, objectName, filePath string, opts GetObjectOptions) error -Identical to FGetObject operation, but allows request cancellation. - -__Parameters__ - - -|Param |Type |Description | -|:---|:---| :---| -|`ctx` | _context.Context_ |Request context | -|`bucketName` | _string_ |Name of the bucket | -|`objectName` | _string_ |Name of the object | -|`filePath` | _string_ |Path to download object to | -|`opts` | _minio.GetObjectOptions_ | Options for GET requests specifying additional options like encryption, If-Match | - - -__Example__ - - -```go -ctx, cancel := context.WithTimeout(context.Background(), 100 * time.Second) -defer cancel() - -err = minioClient.FGetObjectWithContext(ctx, "mybucket", "myobject", "/tmp/myobject", minio.GetObjectOptions{}) +err = minioClient.FGetObject(context.Background(), "mybucket", "myobject", "/tmp/myobject", minio.GetObjectOptions{}) if err != nil { fmt.Println(err) return @@ -660,7 +588,7 @@ if err != nil { ``` -### PutObject(bucketName, objectName string, reader io.Reader, objectSize int64,opts PutObjectOptions) (n int, err error) +### PutObject(ctx context.Context, bucketName, objectName string, reader io.Reader, objectSize int64,opts PutObjectOptions) (n int, err error) Uploads objects that are less than 128MiB in a single PUT operation. For objects that are greater than 128MiB in size, PutObject seamlessly uploads the object as parts of 128MiB or more depending on the actual file size. The max upload size for an object is 5TB. __Parameters__ @@ -668,6 +596,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | |`objectName` | _string_ |Name of the object | |`reader` | _io.Reader_ |Any Go type that implements io.Reader | @@ -688,7 +617,7 @@ __minio.PutObjectOptions__ | `opts.CacheControl` | _string_ | Used to specify directives for caching mechanisms in both requests and responses e.g "max-age=600" | | `opts.Mode` | _*minio.RetentionMode_ | Retention mode to be set, e.g "COMPLIANCE" | | `opts.RetainUntilDate` | _*time.Time_ | Time until which the retention applied is valid | -| `opts.ServerSideEncryption` | _encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v6) | +| `opts.ServerSideEncryption` | _encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v7) | | `opts.StorageClass` | _string_ | Specify storage class for the object. Supported values for MinIO server are `REDUCED_REDUNDANCY` and `STANDARD` | | `opts.WebsiteRedirectLocation` | _string_ | Specify a redirect for the object, to another object in the same bucket or to a external URL. | | `opts.SendContentMd5` | _bool_ | Specify if you'd like to send `content-md5` header with PutObject operation. Note that setting this flag will cause higher memory usage because of in-memory `md5sum` calculation. | @@ -712,7 +641,7 @@ if err != nil { return } -n, err := minioClient.PutObject("mybucket", "myobject", file, fileStat.Size(), minio.PutObjectOptions{ContentType:"application/octet-stream"}) +n, err := minioClient.PutObject(context.Background(), "mybucket", "myobject", file, fileStat.Size(), minio.PutObjectOptions{ContentType:"application/octet-stream"}) if err != nil { fmt.Println(err) return @@ -722,55 +651,9 @@ fmt.Println("Successfully uploaded bytes: ", n) API methods PutObjectWithSize, PutObjectWithMetadata, PutObjectStreaming, and PutObjectWithProgress available in minio-go SDK release v3.0.3 are replaced by the new PutObject call variant that accepts a pointer to PutObjectOptions struct. - -### PutObjectWithContext(ctx context.Context, bucketName, objectName string, reader io.Reader, objectSize int64, opts PutObjectOptions) (n int, err error) -Identical to PutObject operation, but allows request cancellation. - -__Parameters__ - - -|Param |Type |Description | -|:---|:---| :---| -|`ctx` | _context.Context_ |Request context | -|`bucketName` | _string_ |Name of the bucket | -|`objectName` | _string_ |Name of the object | -|`reader` | _io.Reader_ |Any Go type that implements io.Reader | -|`objectSize`| _int64_ | size of the object being uploaded. Pass -1 if stream size is unknown | -|`opts` | _minio.PutObjectOptions_ |Pointer to struct that allows user to set optional custom metadata, content-type, content-encoding, content-disposition, content-language and cache-control headers, pass encryption module for encrypting objects, and optionally configure number of threads for multipart put operation. | - - -__Example__ - - -```go -ctx, cancel := context.WithTimeout(context.Background(), 10 * time.Second) -defer cancel() - -file, err := os.Open("my-testfile") -if err != nil { - fmt.Println(err) - return -} -defer file.Close() - -fileStat, err := file.Stat() -if err != nil { - fmt.Println(err) - return -} - -n, err := minioClient.PutObjectWithContext(ctx, "my-bucketname", "my-objectname", file, fileStat.Size(), minio.PutObjectOptions{ - ContentType: "application/octet-stream", -}) -if err != nil { - fmt.Println(err) - return -} -fmt.Println("Successfully uploaded bytes: ", n) -``` -### CopyObject(dst DestinationInfo, src SourceInfo) error +### CopyObject(ctx context.Context, dst DestinationInfo, src SourceInfo) error Create or replace an object through server-side copying of an existing object. It supports conditional copying, copying a part of an object and server-side encryption of destination and decryption of source. See the `SourceInfo` and `DestinationInfo` types for further details. To copy multiple source objects into a single destination object see the `ComposeObject` API. @@ -780,6 +663,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`dst` | _minio.DestinationInfo_ |Argument describing the destination object | |`src` | _minio.SourceInfo_ |Argument describing the source object | @@ -800,7 +684,7 @@ if err != nil { } // Copy object call -err = minioClient.CopyObject(dst, src) +err = minioClient.CopyObject(context.Background(), dst, src) if err != nil { fmt.Println(err) return @@ -838,7 +722,7 @@ if err != nil { } // Copy object call -err = minioClient.CopyObject(dst, src) +err = minioClient.CopyObject(context.Background(), dst, src) if err != nil { fmt.Println(err) return @@ -846,7 +730,7 @@ if err != nil { ``` -### ComposeObject(dst minio.DestinationInfo, srcs []minio.SourceInfo) error +### ComposeObject(ctx context.Context, dst minio.DestinationInfo, srcs []minio.SourceInfo) error Create an object by concatenating a list of source objects using server-side copying. __Parameters__ @@ -854,6 +738,7 @@ __Parameters__ |Param |Type |Description | |:---|:---|:---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`dst` | _minio.DestinationInfo_ |Struct with info about the object to be created. | |`srcs` | _[]minio.SourceInfo_ |Slice of struct with info about source objects to be concatenated in order. | @@ -891,7 +776,7 @@ if err != nil { } // Compose object call by concatenating multiple source files. -err = minioClient.ComposeObject(dst, srcs) +err = minioClient.ComposeObject(context.Background(), dst, srcs) if err != nil { fmt.Println(err) return @@ -910,7 +795,7 @@ __Parameters__ | :--- | :--- | :--- | | `bucket` | _string_ | Name of the source bucket | | `object` | _string_ | Name of the source object | -| `sse` | _*encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v6) | +| `sse` | _*encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v7) | __Example__ @@ -963,7 +848,7 @@ __Parameters__ | :--- | :--- | :--- | | `bucket` | _string_ | Name of the destination bucket | | `object` | _string_ | Name of the destination object | -| `sse` | _*encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v6) | | +| `sse` | _*encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v7) | | | `userMeta` | _map[string]string_ | User metadata to be set on the destination. If nil, with only one source, user-metadata is copied from source. | __Example__ @@ -1020,7 +905,7 @@ __minio.DestInfoOptions__ |Field | Type | Description | |:--- |:--- | :--- | -| `destOpts.Encryption` | _encrypt.ServerSide_ | Interface provided by encrypt package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v6). | +| `destOpts.Encryption` | _encrypt.ServerSide_ | Interface provided by encrypt package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v7). | | `destOpts.UserMetadata` | _map[string]string_ | Map of user meta data to be set on destination object. | | `destOpts.UserTags` | _map[string]string_ | Map of user object tags to be set on destination object. | | `destOpts.ReplaceTags` | _bool_ | Replace object tags of the destination object. | @@ -1046,7 +931,7 @@ if err != nil { } // Copy object call -err = minioClient.CopyObject(dst, src) +err = minioClient.CopyObject(context.Background(), dst, src) if err != nil { fmt.Println(err) return @@ -1071,7 +956,7 @@ if err != nil { } // Copy object call -err = minioClient.CopyObject(dst, src) +err = minioClient.CopyObject(context.Background(), dst, src) if err != nil { fmt.Println(err) return @@ -1079,7 +964,7 @@ if err != nil { ``` -### FPutObject(bucketName, objectName, filePath, opts PutObjectOptions) (length int64, err error) +### FPutObject(ctx context.Context, bucketName, objectName, filePath, opts PutObjectOptions) (length int64, err error) Uploads contents from a file to objectName. FPutObject uploads objects that are less than 128MiB in a single PUT operation. For objects that are greater than the 128MiB in size, FPutObject seamlessly uploads the object in chunks of 128MiB or more depending on the actual file size. The max upload size for an object is 5TB. @@ -1089,6 +974,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | |`objectName` | _string_ |Name of the object | |`filePath` | _string_ |Path to file to be uploaded | @@ -1099,7 +985,7 @@ __Example__ ```go -n, err := minioClient.FPutObject("my-bucketname", "my-objectname", "my-filename.csv", minio.PutObjectOptions{ +n, err := minioClient.FPutObject(context.Background(), "my-bucketname", "my-objectname", "my-filename.csv", minio.PutObjectOptions{ ContentType: "application/csv", }); if err != nil { @@ -1109,38 +995,8 @@ if err != nil { fmt.Println("Successfully uploaded bytes: ", n) ``` - -### FPutObjectWithContext(ctx context.Context, bucketName, objectName, filePath, opts PutObjectOptions) (length int64, err error) -Identical to FPutObject operation, but allows request cancellation. - -__Parameters__ - - -|Param |Type |Description | -|:---|:---| :---| -|`ctx` | _context.Context_ |Request context | -|`bucketName` | _string_ |Name of the bucket | -|`objectName` | _string_ |Name of the object | -|`filePath` | _string_ |Path to file to be uploaded | -|`opts` | _minio.PutObjectOptions_ |Pointer to struct that allows user to set optional custom metadata, content-type, content-encoding,content-disposition and cache-control headers, pass encryption module for encrypting objects, and optionally configure number of threads for multipart put operation. | - -__Example__ - - -```go -ctx, cancel := context.WithTimeout(context.Background(), 100 * time.Second) -defer cancel() - -n, err := minioClient.FPutObjectWithContext(ctx, "mybucket", "myobject.csv", "/tmp/otherobject.csv", minio.PutObjectOptions{ContentType:"application/csv"}) -if err != nil { - fmt.Println(err) - return -} -fmt.Println("Successfully uploaded bytes: ", n) -``` - -### StatObject(bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) +### StatObject(ctx context.Context, bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) Fetch metadata of an object. __Parameters__ @@ -1148,6 +1004,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | |`objectName` | _string_ |Name of the object | |`opts` | _minio.StatObjectOptions_ | Options for GET info/stat requests specifying additional options like encryption, If-Match | @@ -1174,7 +1031,7 @@ __Example__ ```go -objInfo, err := minioClient.StatObject("mybucket", "myobject", minio.StatObjectOptions{}) +objInfo, err := minioClient.StatObject(context.Background(), "mybucket", "myobject", minio.StatObjectOptions{}) if err != nil { fmt.Println(err) return @@ -1183,7 +1040,7 @@ fmt.Println(objInfo) ``` -### RemoveObject(bucketName, objectName string) error +### RemoveObject(ctx context.Context, bucketName, objectName string) error Removes an object. __Parameters__ @@ -1191,12 +1048,13 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | |`objectName` | _string_ |Name of the object | ```go -err = minioClient.RemoveObject("mybucket", "myobject") +err = minioClient.RemoveObject(context.Background(), "mybucket", "myobject") if err != nil { fmt.Println(err) return @@ -1204,13 +1062,14 @@ if err != nil { ``` -### RemoveObjects(bucketName string, objectsCh chan string) (errorCh <-chan RemoveObjectError) +### RemoveObjects(ctx context.Context, bucketName string, objectsCh chan string) (errorCh <-chan RemoveObjectError) Removes a list of objects obtained from an input channel. The call sends a delete request to the server up to 1000 objects at a time. The errors observed are sent over the error channel. __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | |`objectsCh` | _chan string_ | Channel of objects to be removed | @@ -1229,7 +1088,7 @@ objectsCh := make(chan string) go func() { defer close(objectsCh) // List all objects from a bucket-name with a matching prefix. - for object := range minioClient.ListObjects("my-bucketname", "my-prefixname", true, nil) { + for object := range minioClient.ListObjects(context.Background(), "my-bucketname", "my-prefixname", true, nil) { if object.Err != nil { log.Fatalln(object.Err) } @@ -1237,60 +1096,21 @@ go func() { } }() -for rErr := range minioClient.RemoveObjects("mybucket", objectsCh) { +for rErr := range minioClient.RemoveObjects(context.Background(), "mybucket", objectsCh) { fmt.Println("Error detected during deletion: ", rErr) } ``` - -### RemoveObjectsWithContext(ctx context.Context, bucketName string, objectsCh chan string) (errorCh <-chan RemoveObjectError) -*Identical to RemoveObjects operation, but accepts a context for request cancellation.* - -Parameters - -|Param |Type |Description | -|:---|:---| :---| -|`ctx` | _context.Context_ |Request context | -|`bucketName` | _string_ |Name of the bucket | -|`objectsCh` | _chan string_ | Channel of objects to be removed | - - -__Return Values__ - -|Param |Type |Description | -|:---|:---| :---| -|`errorCh` | _<-chan minio.RemoveObjectError_ | Receive-only channel of errors observed during deletion. | - -```go -objectsCh := make(chan string) -ctx, cancel := context.WithTimeout(context.Background(), 100 * time.Second) -defer cancel() - -// Send object names that are needed to be removed to objectsCh -go func() { - defer close(objectsCh) - // List all objects from a bucket-name with a matching prefix. - for object := range minioClient.ListObjects("my-bucketname", "my-prefixname", true, nil) { - if object.Err != nil { - log.Fatalln(object.Err) - } - objectsCh <- object.Key - } -}() - -for rErr := range minioClient.RemoveObjects(ctx, "my-bucketname", objectsCh) { - fmt.Println("Error detected during deletion: ", rErr) -} -``` -### RemoveObjectWithOptions(bucketName, objectName string, opts minio.RemoveObjectOptions) error -Removes an object. +### RemoveObjectWithOptions(ctx context.Context, bucketName, objectName string, opts minio.RemoveObjectOptions) error +Removes an object with more specified options __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | |`objectName` | _string_ |Name of the object | |`opts` |_minio.RemoveObjectOptions_ |Allows user to set options | @@ -1308,14 +1128,14 @@ opts := minio.RemoveObjectOptions { GovernanceBypass: true, VersionID: "myversionid", } -err = minioClient.RemoveObjectWithOptions("mybucket", "myobject", opts) +err = minioClient.RemoveObjectWithOptions(context.Background(), "mybucket", "myobject", opts) if err != nil { fmt.Println(err) return } ``` -### PutObjectRetention(bucketName, objectName string, opts minio.PutObjectRetentionOptions) error +### PutObjectRetention(ctx context.Context, bucketName, objectName string, opts minio.PutObjectRetentionOptions) error Applies object retention lock onto an object. __Parameters__ @@ -1323,18 +1143,20 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | |`objectName` | _string_ |Name of the object | |`opts` |_minio.PutObjectRetentionOptions_ |Allows user to set options like retention mode, expiry date and version id | -### RemoveObjectsWithOptions(bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError +### RemoveObjectsWithOptions(ctx context.Context, bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError *Identical to RemoveObjects operation, but accepts opts for bypassing Governance mode.* Parameters |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | |`objectsCh` | _chan string_ | Channel of objects to be removed | |`opts` |_minio.RemoveObjectsOptions_ | Allows user to set options | @@ -1358,52 +1180,7 @@ objectsCh := make(chan string) go func() { defer close(objectsCh) // List all objects from a bucket-name with a matching prefix. - for object := range minioClient.ListObjects("my-bucketname", "my-prefixname", true, nil) { - if object.Err != nil { - log.Fatalln(object.Err) - } - objectsCh <- object.Key - } -}() - -opts := minio.RemoveObjectsOptions{ - GovernanceBypass: true, -} - -for rErr := range minioClient.RemoveObjectsWithOptions("my-bucketname", objectsCh, opts) { - fmt.Println("Error detected during deletion: ", rErr) -} -``` - - -### RemoveObjectsWithOptionsContext(ctx context.Context, bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError -*Identical to RemoveObjectsWithContext operation, but accepts opts for bypassing Governance mode.* - -Parameters - -|Param |Type |Description | -|:---|:---| :---| -|`ctx` | _context.Context_ |Request context | -|`bucketName` | _string_ |Name of the bucket | -|`objectsCh` | _chan string_ | Channel of objects to be removed | -|`opts` |_minio.RemoveObjectsOptions_ | Allows user to set options | - -__Return Values__ - -|Param |Type |Description | -|:---|:---| :---| -|`errorCh` | _<-chan minio.RemoveObjectError_ | Receive-only channel of errors observed during deletion. | - -```go -objectsCh := make(chan string) -ctx, cancel := context.WithTimeout(context.Background(), 100 * time.Second) -defer cancel() - -// Send object names that are needed to be removed to objectsCh -go func() { - defer close(objectsCh) - // List all objects from a bucket-name with a matching prefix. - for object := range minioClient.ListObjects("my-bucketname", "my-prefixname", true, nil) { + for object := range minioClient.ListObjects(context.Background(), "my-bucketname", "my-prefixname", true, nil) { if object.Err != nil { log.Fatalln(object.Err) } @@ -1415,36 +1192,13 @@ opts := minio.RemoveObjectsOptions{ GovernanceBypass: true, } -for rErr := range minioClient.RemoveObjectsWithOptionsContext(ctx, "my-bucketname", objectsCh, opts) { +for rErr := range minioClient.RemoveObjectsWithOptions(context.Background(), "my-bucketname", objectsCh, opts) { fmt.Println("Error detected during deletion: ", rErr) } ``` -__minio.PutObjectRetentionOptions__ - -|Field | Type | Description | -|:--- |:--- | :--- | -| `opts.GovernanceBypass` | _bool_ |Set the bypass governance header to overwrite object retention if the existing retention mode is set to GOVERNANCE| -| `opts.Mode` | _*minio.RetentionMode_ |Retention mode to be set| -| `opts.RetainUntilDate` | _*time.Time_ |Time until which the retention applied is valid| -| `opts.VersionID` | _string_ |Version ID of the object to apply retention on| - -```go -t := time.Date(2020, time.November, 18, 14, 0, 0, 0, time.UTC) -m := minio.RetentionMode(minio.Compliance) -opts := minio.PutObjectRetentionOptions { - GovernanceBypass: true, - RetainUntilDate: &t, - Mode: &m, - } -err = minioClient.PutObjectRetention("mybucket", "myobject", opts) -if err != nil { - fmt.Println(err) - return -} -``` -### GetObjectRetention(bucketName, objectName, versionID string) (mode *RetentionMode, retainUntilDate *time.Time, err error) +### GetObjectRetention(ctx context.Context, bucketName, objectName, versionID string) (mode *RetentionMode, retainUntilDate *time.Time, err error) Returns retention set on a given object. __Parameters__ @@ -1452,19 +1206,20 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | |`objectName` | _string_ |Name of the object | |`versionID` |_string_ |Version ID of the object | ```go -err = minioClient.PutObjectRetention("mybucket", "myobject", "") +err = minioClient.PutObjectRetention(context.Background(), "mybucket", "myobject", "") if err != nil { fmt.Println(err) return } ``` -### PutObjectLegalHold(bucketName, objectName string, opts minio.PutObjectLegalHoldOptions) error +### PutObjectLegalHold(ctx context.Context, bucketName, objectName string, opts minio.PutObjectLegalHoldOptions) error Applies legal-hold onto an object. __Parameters__ @@ -1472,6 +1227,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | |`objectName` | _string_ |Name of the object | |`opts` |_minio.PutObjectLegalHoldOptions_ |Allows user to set options like status and version id | @@ -1488,27 +1244,28 @@ s := minio.LegalHoldEnabled opts := minio.PutObjectLegalHoldOptions { Status: &s, } -err = minioClient.PutObjectLegalHold("mybucket", "myobject", opts) +err = minioClient.PutObjectLegalHold(context.Background(), "mybucket", "myobject", opts) if err != nil { fmt.Println(err) return } ``` -### GetObjectLegalHold(bucketName, objectName, versionID string) (status *LegalHoldStatus, err error) +### GetObjectLegalHold(ctx context.Context, bucketName, objectName, versionID string) (status *LegalHoldStatus, err error) Returns legal-hold status on a given object. __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | |`objectName` | _string_ |Name of the object | |`opts` |_minio.GetObjectLegalHoldOptions_ |Allows user to set options like version id | ```go opts := minio.GetObjectLegalHoldOptions{} -err = minioClient.GetObjectLegalHold("mybucket", "myobject", opts) +err = minioClient.GetObjectLegalHold(context.Background(), "mybucket", "myobject", opts) if err != nil { fmt.Println(err) return @@ -1520,6 +1277,7 @@ Parameters |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`ctx` | _context.Context_ |Request context | |`bucketName` | _string_ |Name of the bucket | |`objectName` | _string_ |Name of the object | @@ -1569,7 +1327,7 @@ __Return Values__ ``` -### PutObjectTagging(bucketName, objectName string, objectTags map[string]string) error +### PutObjectTagging(ctx context.Context, bucketName, objectName string, objectTags map[string]string) error Adds or replace Object Tags to the given object __Parameters__ @@ -1577,6 +1335,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | |`objectName` | _string_ |Name of the object | |`objectTags` | _map[string]string_ | Map with Object Tag's Key and Value | @@ -1585,32 +1344,7 @@ __Example__ ```go -err = minioClient.PutObjectTagging(bucketName, objectName, objectTags) -if err != nil { - fmt.Println(err) - return -} -``` - - -### PutObjectTaggingWithContext(ctx context.Context, sssbucketName, objectName string, objectTags map[string]string) error -Identical to PutObjectTagging, but allows setting context to allow controlling context cancellations and timeouts. - -__Parameters__ - - -|Param |Type |Description | -|:---|:---| :---| -|`ctx` | _context.Context_ |Request context | -|`bucketName` | _string_ |Name of the bucket | -|`objectName` | _string_ |Name of the object | -|`objectTags` | _map[string]string_ | Map with Object Tag's Key and Value | - -__Example__ - - -```go -err = minioClient.PutObjectTaggingWithContext(ctx, bucketName, objectName, objectTags) +err = minioClient.PutObjectTagging(context.Background(), bucketName, objectName, objectTags) if err != nil { fmt.Println(err) return @@ -1618,7 +1352,7 @@ if err != nil { ``` -### GetObjectTagging(bucketName, objectName string) (string, error) +### GetObjectTagging(ctx context.Context, bucketName, objectName string) (string, error) Fetch Object Tags from the given object __Parameters__ @@ -1626,6 +1360,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | |`objectName` | _string_ |Name of the object | @@ -1633,32 +1368,7 @@ __Example__ ```go -tags, err = minioClient.GetObjectTagging(bucketName, objectName) -if err != nil { - fmt.Println(err) - return -} -fmt.Printf("Fetched Tags: %s", tags) -``` - - -### GetObjectTaggingWithContext(ctx context.Context, bucketName, objectName string) (string, error) -Identical to GetObjectTagging, but allows setting context to allow controlling context cancellations and timeouts. - -__Parameters__ - - -|Param |Type |Description | -|:---|:---| :---| -|`ctx` | _context.Context_ |Request context | -|`bucketName` | _string_ |Name of the bucket | -|`objectName` | _string_ |Name of the object | - -__Example__ - - -```go -tags, err = minioClient.GetObjectTaggingWithContext(ctx, bucketName, objectName) +tags, err = minioClient.GetObjectTagging(context.Background(), bucketName, objectName) if err != nil { fmt.Println(err) return @@ -1667,7 +1377,7 @@ fmt.Printf("Fetched Tags: %s", tags) ``` -### RemoveObjectTagging(bucketName, objectName string) error +### RemoveObjectTagging(ctx context.Context, bucketName, objectName string) error Remove Object Tags from the given object __Parameters__ @@ -1675,6 +1385,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | |`objectName` | _string_ |Name of the object | @@ -1682,31 +1393,7 @@ __Example__ ```go -err = minioClient.RemoveObjectTagging(bucketName, objectName) -if err != nil { - fmt.Println(err) - return -} -``` - - -### RemoveObjectTaggingWithContext(ctx context.Context, bucketName, objectName string) error -Identical to RemoveObjectTagging, but allows setting context to allow controlling context cancellations and timeouts. - -__Parameters__ - - -|Param |Type |Description | -|:---|:---| :---| -|`ctx` | _context.Context_ |Request context | -|`bucketName` | _string_ |Name of the bucket | -|`objectName` | _string_ |Name of the object | - -__Example__ - - -```go -err = minioClient.RemoveObjectTaggingWithContext(ctx, bucketName, objectName) +err = minioClient.RemoveObjectTagging(context.Background(), bucketName, objectName) if err != nil { fmt.Println(err) return @@ -1714,7 +1401,7 @@ if err != nil { ``` -### RemoveIncompleteUpload(bucketName, objectName string) error +### RemoveIncompleteUpload(ctx context.Context, bucketName, objectName string) error Removes a partially uploaded object. __Parameters__ @@ -1722,6 +1409,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | |`objectName` | _string_ |Name of the object | @@ -1729,7 +1417,7 @@ __Example__ ```go -err = minioClient.RemoveIncompleteUpload("mybucket", "myobject") +err = minioClient.RemoveIncompleteUpload(context.Background(), "mybucket", "myobject") if err != nil { fmt.Println(err) return @@ -1739,7 +1427,7 @@ if err != nil { ## 5. Presigned operations -### PresignedGetObject(bucketName, objectName string, expiry time.Duration, reqParams url.Values) (*url.URL, error) +### PresignedGetObject(ctx context.Context, bucketName, objectName string, expiry time.Duration, reqParams url.Values) (*url.URL, error) Generates a presigned URL for HTTP GET operations. Browsers/Mobile clients may point to this URL to directly download objects even if the bucket is private. This presigned URL can have an associated expiration time in seconds after which it is no longer operational. The default expiry is set to 7 days. __Parameters__ @@ -1747,6 +1435,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | |`objectName` | _string_ |Name of the object | |`expiry` | _time.Duration_ |Expiry of presigned URL in seconds | @@ -1762,7 +1451,7 @@ reqParams := make(url.Values) reqParams.Set("response-content-disposition", "attachment; filename=\"your-filename.txt\"") // Generates a presigned url which expires in a day. -presignedURL, err := minioClient.PresignedGetObject("mybucket", "myobject", time.Second * 24 * 60 * 60, reqParams) +presignedURL, err := minioClient.PresignedGetObject(context.Background(), "mybucket", "myobject", time.Second * 24 * 60 * 60, reqParams) if err != nil { fmt.Println(err) return @@ -1771,7 +1460,7 @@ fmt.Println("Successfully generated presigned URL", presignedURL) ``` -### PresignedPutObject(bucketName, objectName string, expiry time.Duration) (*url.URL, error) +### PresignedPutObject(ctx context.Context, bucketName, objectName string, expiry time.Duration) (*url.URL, error) Generates a presigned URL for HTTP PUT operations. Browsers/Mobile clients may point to this URL to upload objects directly to a bucket even if it is private. This presigned URL can have an associated expiration time in seconds after which it is no longer operational. The default expiry is set to 7 days. NOTE: you can upload to S3 only with specified object name. @@ -1781,6 +1470,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | |`objectName` | _string_ |Name of the object | |`expiry` | _time.Duration_ |Expiry of presigned URL in seconds | @@ -1792,7 +1482,7 @@ __Example__ ```go // Generates a url which expires in a day. expiry := time.Second * 24 * 60 * 60 // 1 day. -presignedURL, err := minioClient.PresignedPutObject("mybucket", "myobject", expiry) +presignedURL, err := minioClient.PresignedPutObject(context.Background(), "mybucket", "myobject", expiry) if err != nil { fmt.Println(err) return @@ -1801,13 +1491,14 @@ fmt.Println("Successfully generated presigned URL", presignedURL) ``` -### PresignedHeadObject(bucketName, objectName string, expiry time.Duration, reqParams url.Values) (*url.URL, error) +### PresignedHeadObject(ctx context.Context, bucketName, objectName string, expiry time.Duration, reqParams url.Values) (*url.URL, error) Generates a presigned URL for HTTP HEAD operations. Browsers/Mobile clients may point to this URL to directly get metadata from objects even if the bucket is private. This presigned URL can have an associated expiration time in seconds after which it is no longer operational. The default expiry is set to 7 days. __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | |`objectName` | _string_ |Name of the object | |`expiry` | _time.Duration_ |Expiry of presigned URL in seconds | @@ -1823,7 +1514,7 @@ reqParams := make(url.Values) reqParams.Set("response-content-disposition", "attachment; filename=\"your-filename.txt\"") // Generates a presigned url which expires in a day. -presignedURL, err := minioClient.PresignedHeadObject("mybucket", "myobject", time.Second * 24 * 60 * 60, reqParams) +presignedURL, err := minioClient.PresignedHeadObject(context.Background(), "mybucket", "myobject", time.Second * 24 * 60 * 60, reqParams) if err != nil { fmt.Println(err) return @@ -1832,7 +1523,7 @@ fmt.Println("Successfully generated presigned URL", presignedURL) ``` -### PresignedPostPolicy(PostPolicy) (*url.URL, map[string]string, error) +### PresignedPostPolicy(ctx context.Context, post PostPolicy) (*url.URL, map[string]string, error) Allows setting policy conditions to a presigned URL for POST operations. Policies such as bucket name to receive object uploads, key name prefixes, expiry policy may be set. ```go @@ -1854,7 +1545,7 @@ policy.SetContentLengthRange(1024, 1024*1024) policy.SetUserMetadata("custom", "user") // Get the POST form key/value object: -url, formData, err := minioClient.PresignedPostPolicy(policy) +url, formData, err := minioClient.PresignedPostPolicy(context.Background(), policy) if err != nil { fmt.Println(err) return @@ -1872,13 +1563,14 @@ fmt.Printf("%s\n", url) ## 6. Bucket policy/notification operations -### SetBucketPolicy(bucketname, policy string) error +### SetBucketPolicy(ctx context.Context, bucketname, policy string) error Set access permissions on bucket or an object prefix. __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket| |`policy` | _string_ |Policy to be set | @@ -1893,7 +1585,7 @@ __Example__ ```go policy := `{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::my-bucketname/*"],"Sid": ""}]}` -err = minioClient.SetBucketPolicy("my-bucketname", policy) +err = minioClient.SetBucketPolicy(context.Background(), "my-bucketname", policy) if err != nil { fmt.Println(err) return @@ -1901,7 +1593,7 @@ if err != nil { ``` -### GetBucketPolicy(bucketName) (policy string, error) +### GetBucketPolicy(ctx context.Context, bucketName string) (policy string, error) Get access permissions on a bucket or a prefix. __Parameters__ @@ -1909,6 +1601,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | __Return Values__ @@ -1922,14 +1615,14 @@ __Return Values__ __Example__ ```go -policy, err := minioClient.GetBucketPolicy("my-bucketname") +policy, err := minioClient.GetBucketPolicy(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } ``` -### GetBucketNotification(bucketName string) (BucketNotification, error) +### GetBucketNotification(ctx context.Context, bucketName string) (BucketNotification, error) Get notification configuration on a bucket. __Parameters__ @@ -1937,6 +1630,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | __Return Values__ @@ -1951,7 +1645,7 @@ __Example__ ```go -bucketNotification, err := minioClient.GetBucketNotification("mybucket") +bucketNotification, err := minioClient.GetBucketNotification(context.Background(), "mybucket") if err != nil { fmt.Println("Failed to get bucket notification configurations for mybucket", err) return @@ -1965,7 +1659,7 @@ for _, queueConfig := range bucketNotification.QueueConfigs { ``` -### SetBucketNotification(bucketName string, bucketNotification BucketNotification) error +### SetBucketNotification(ctx context.Context, bucketName string, bucketNotification BucketNotification) error Set a new bucket notification on a bucket. __Parameters__ @@ -1973,6 +1667,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | |`bucketNotification` | _minio.BucketNotification_ |Represents the XML to be sent to the configured web service | @@ -1997,7 +1692,7 @@ queueConfig.AddFilterSuffix(".jpg") bucketNotification := minio.BucketNotification{} bucketNotification.AddQueue(queueConfig) -err = minioClient.SetBucketNotification("mybucket", bucketNotification) +err = minioClient.SetBucketNotification(context.Background(), "mybucket", bucketNotification) if err != nil { fmt.Println("Unable to set the bucket notification: ", err) return @@ -2005,7 +1700,7 @@ if err != nil { ``` -### RemoveAllBucketNotification(bucketName string) error +### RemoveAllBucketNotification(ctx context.Context, bucketName string) error Remove all configured bucket notifications on a bucket. __Parameters__ @@ -2013,6 +1708,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | __Return Values__ @@ -2026,7 +1722,7 @@ __Example__ ```go -err = minioClient.RemoveAllBucketNotification("mybucket") +err = minioClient.RemoveAllBucketNotification(context.Background(), "mybucket") if err != nil { fmt.Println("Unable to remove bucket notifications.", err) return @@ -2034,7 +1730,7 @@ if err != nil { ``` -### ListenBucketNotification(bucketName, prefix, suffix string, events []string, doneCh <-chan struct{}) <-chan NotificationInfo +### ListenBucketNotification(context context.Context, bucketName, prefix, suffix string, events []string, doneCh <-chan struct{}) <-chan NotificationInfo ListenBucketNotification API receives bucket notification events through the notification channel. The returned notification channel has two fields 'Records' and 'Err'. - 'Records' holds the notifications received from the server. @@ -2077,7 +1773,7 @@ doneCh := make(chan struct{}) defer close(doneCh) // Listen for bucket notifications on "mybucket" filtered by prefix, suffix and events. -for notificationInfo := range minioClient.ListenBucketNotification("mybucket", "myprefix/", ".mysuffix", []string{ +for notificationInfo := range minioClient.ListenBucketNotification(context.Background(), "mybucket", "myprefix/", ".mysuffix", []string{ "s3:ObjectCreated:*", "s3:ObjectAccessed:*", "s3:ObjectRemoved:*", @@ -2090,13 +1786,14 @@ for notificationInfo := range minioClient.ListenBucketNotification("mybucket", " ``` -### SetBucketLifecycle(bucketname, lifecycle string) error +### SetBucketLifecycle(ctx context.Context, bucketname, lifecycle string) error Set lifecycle on bucket or an object prefix. __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket| |`lifecycle` | _string_ |Lifecycle to be set | @@ -2120,7 +1817,7 @@ lifecycle := ` ` -err = minioClient.SetBucketLifecycle("my-bucketname", lifecycle) +err = minioClient.SetBucketLifecycle(context.Background(), "my-bucketname", lifecycle) if err != nil { fmt.Println(err) return @@ -2128,7 +1825,7 @@ if err != nil { ``` -### GetBucketLifecycle(bucketName) (lifecycle string, error) +### GetBucketLifecycle(ctx context.Context, bucketName string) (lifecycle string, error) Get lifecycle on a bucket or a prefix. __Parameters__ @@ -2136,6 +1833,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | __Return Values__ @@ -2149,20 +1847,21 @@ __Return Values__ __Example__ ```go -lifecycle, err := minioClient.GetBucketLifecycle("my-bucketname") +lifecycle, err := minioClient.GetBucketLifecycle(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } ``` -### SetBucketEncryption(bucketname string, configuration ServerSideEncryptionConfiguration) error +### SetBucketEncryption(ctx context.Context, bucketname string, configuration ServerSideEncryptionConfiguration) error Set default encryption configuration on a bucket. __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket| |`configuration` | _minio.ServerSideEncyrptionConfiguration_ | Structure that holds default encryption configuration to be set | @@ -2189,14 +1888,14 @@ config := minio.ServerSideEncryptionConfiguration{Rules: []minio.Rule{ }, }} // Set default encryption configuration on an S3 bucket -err = s3Client.SetBucketEncryption("my-bucketname", config) +err = s3Client.SetBucketEncryption(context.Background(), "my-bucketname", config) if err != nil { log.Fatalln(err) } ``` -### GetBucketEncryption(bucketName string) (ServerSideEncryptionConfiguration, error) +### GetBucketEncryption(ctx context.Context, bucketName string) (ServerSideEncryptionConfiguration, error) Get default encryption configuration set on a bucket. __Parameters__ @@ -2204,6 +1903,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | __Return Values__ @@ -2223,7 +1923,7 @@ if err != nil { } // Get default encryption configuration set on an S3 bucket and print it out -encryptionConfig, err := s3Client.GetBucketEncryption("my-bucketname") +encryptionConfig, err := s3Client.GetBucketEncryption(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } @@ -2231,7 +1931,7 @@ fmt.Printf("%+v\n", encryptionConfig) ``` -### DeleteBucketEncryption(bucketName string) (error) +### DeleteBucketEncryption(ctx context.Context, bucketName string) (error) Delete/Remove default encryption configuration set on a bucket. __Parameters__ @@ -2239,6 +1939,7 @@ __Parameters__ |Param |Type |Description | |:---|:---|:---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | __Return Values__ @@ -2251,7 +1952,7 @@ __Return Values__ __Example__ ```go -err := s3Client.DeleteBucketEncryption("my-bucketname") +err := s3Client.DeleteBucketEncryption(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } @@ -2259,13 +1960,14 @@ if err != nil { ``` -### SetObjectLockConfig(bucketname, mode *RetentionMode, validity *uint, unit *ValidityUnit) error +### SetObjectLockConfig(ctx context.Context, bucketname, mode *RetentionMode, validity *uint, unit *ValidityUnit) error Set object lock configuration in given bucket. mode, validity and unit are either all set or all nil. __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket| |`mode` | _RetentionMode_ |Retention mode to be set | |`validity` | _uint_ |Validity period to be set | @@ -2284,7 +1986,7 @@ mode := Governance validity := uint(30) unit := Days -err = minioClient.SetObjectLockConfig("my-bucketname", &mode, &validity, &unit) +err = minioClient.SetObjectLockConfig(context.Background(), "my-bucketname", &mode, &validity, &unit) if err != nil { fmt.Println(err) return @@ -2292,7 +1994,7 @@ if err != nil { ``` -### GetObjectLockConfig(bucketName) (objectLock,*RetentionMode, *uint, *ValidityUnit, error) +### GetObjectLockConfig(ctx context.Context, bucketName string) (objectLock,*RetentionMode, *uint, *ValidityUnit, error) Get object lock configuration of given bucket. __Parameters__ @@ -2300,6 +2002,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | __Return Values__ @@ -2316,7 +2019,7 @@ __Return Values__ __Example__ ```go -enabled, mode, validity, unit, err := minioClient.GetObjectLockConfig("my-bucketname") +enabled, mode, validity, unit, err := minioClient.GetObjectLockConfig(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } @@ -2329,7 +2032,7 @@ if mode != nil { ``` -### EnableVersioning(bucketName) error +### EnableVersioning(ctx context.Context, bucketName string) error Enable bucket versioning support. __Parameters__ @@ -2337,6 +2040,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | __Return Values__ @@ -2349,7 +2053,7 @@ __Return Values__ __Example__ ```go -err := minioClient.EnableVersioning("my-bucketname") +err := minioClient.EnableVersioning(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } @@ -2358,7 +2062,7 @@ fmt.Println("versioning enabled for bucket 'my-bucketname'") ``` -### DisableVersioning(bucketName) error +### DisableVersioning(ctx context.Context, bucketName) error Disable bucket versioning support. __Parameters__ @@ -2366,6 +2070,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | __Return Values__ @@ -2378,7 +2083,7 @@ __Return Values__ __Example__ ```go -err := minioClient.DisableVersioning("my-bucketname") +err := minioClient.DisableVersioning(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } @@ -2387,7 +2092,7 @@ fmt.Println("versioning disabled for bucket 'my-bucketname'") ``` -### GetBucketVersioning(bucketName string) (BucketVersioningConfiguration, error) +### GetBucketVersioning(ctx context.Context, bucketName string) (BucketVersioningConfiguration, error) Get versioning configuration set on a bucket. __Parameters__ @@ -2395,6 +2100,7 @@ __Parameters__ |Param |Type |Description | |:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | __Return Values__ @@ -2414,7 +2120,7 @@ if err != nil { } // Get versioning configuration set on an S3 bucket and print it out -versioningConfig, err := s3Client.GetBucketVersioning("my-bucketname") +versioningConfig, err := s3Client.GetBucketVersioning(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } diff --git a/docs/zh_CN/API.md b/docs/zh_CN/API.md index eb81b6523..e799d54da 100644 --- a/docs/zh_CN/API.md +++ b/docs/zh_CN/API.md @@ -10,7 +10,7 @@ package main import ( "fmt" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -34,7 +34,7 @@ package main import ( "fmt" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -393,7 +393,7 @@ __minio.GetObjectOptions__ |参数 | 类型 | 描述 | |:---|:---|:---| -| `opts.Materials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口,(更多信息,请看https://godoc.org/github.com/minio/minio-go/v6) | +| `opts.Materials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口,(更多信息,请看https://godoc.org/github.com/minio/minio-go/v7) | __返回值__ @@ -538,7 +538,7 @@ __参数__ |`bucketName` | _string_ |存储桶名称 | |`objectName` | _string_ |对象的名称 | |`filePath` | _string_ |下载后保存的路径| -|`materials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口,(更多信息,请看https://godoc.org/github.com/minio/minio-go/v6) | +|`materials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口,(更多信息,请看https://godoc.org/github.com/minio/minio-go/v7) | __示例__ @@ -587,7 +587,7 @@ __minio.PutObjectOptions__ | `opts.ContentEncoding` | _string_ | 对象的Content encoding,例如"gzip" | | `opts.ContentDisposition` | _string_ | 对象的Content disposition, "inline" | | `opts.CacheControl` | _string_ | 指定针对请求和响应的缓存机制,例如"max-age=600"| -| `opts.EncryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口,(更多信息,请看https://godoc.org/github.com/minio/minio-go/v6) | +| `opts.EncryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口,(更多信息,请看https://godoc.org/github.com/minio/minio-go/v7) | __示例__ @@ -1190,7 +1190,7 @@ __参数__ |:---|:---| :---| |`bucketName` | _string_ | 存储桶名称 | |`objectName` | _string_ | 对象的名称 | -|`encryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口,(更多信息,请看https://godoc.org/github.com/minio/minio-go/v6) | +|`encryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口,(更多信息,请看https://godoc.org/github.com/minio/minio-go/v7) | __返回值__ @@ -1247,7 +1247,7 @@ __参数__ |`bucketName` | _string_ |存储桶名称 | |`objectName` | _string_ |对象的名称 | |`reader` | _io.Reader_ |任何实现io.Reader的Go类型 | -|`encryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口,(更多信息,请看https://godoc.org/github.com/minio/minio-go/v6) | +|`encryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口,(更多信息,请看https://godoc.org/github.com/minio/minio-go/v7) | __示例__ @@ -1309,7 +1309,7 @@ __参数__ |`bucketName` | _string_ |存储桶名称 | |`objectName` | _string_ |对象的名称 | |`filePath` | _string_ |要上传的文件的路径 | -|`encryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口,(更多信息,请看https://godoc.org/github.com/minio/minio-go/v6) | +|`encryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口,(更多信息,请看https://godoc.org/github.com/minio/minio-go/v7) | __示例__ @@ -1503,7 +1503,7 @@ fmt.Printf("%s\n", url) ### SetBucketPolicy(bucketname, objectPrefix string, policy policy.BucketPolicy) error 给存储桶或者对象前缀设置访问权限。 -必须引入`github.com/minio/minio-go/v6/pkg/policy`包。 +必须引入`github.com/minio/minio-go/v7/pkg/policy`包。 __参数__ @@ -1544,7 +1544,7 @@ if err != nil { ### GetBucketPolicy(bucketName, objectPrefix string) (policy.BucketPolicy, error) 获取存储桶或者对象前缀的访问权限。 -必须引入`github.com/minio/minio-go/v6/pkg/policy`包。 +必须引入`github.com/minio/minio-go/v7/pkg/policy`包。 __参数__ diff --git a/examples/minio/listenbucketnotification.go b/examples/minio/listenbucketnotification.go index a899c1407..d5aab5d40 100644 --- a/examples/minio/listenbucketnotification.go +++ b/examples/minio/listenbucketnotification.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -48,7 +49,7 @@ func main() { defer close(doneCh) // Listen for bucket notifications on "mybucket" filtered by prefix, suffix and events. - for notificationInfo := range minioClient.ListenBucketNotification("YOUR-BUCKET", "PREFIX", "SUFFIX", []string{ + for notificationInfo := range minioClient.ListenBucketNotification(context.Background(), "YOUR-BUCKET", "PREFIX", "SUFFIX", []string{ "s3:ObjectCreated:*", "s3:ObjectAccessed:*", "s3:ObjectRemoved:*", diff --git a/examples/s3/bucketexists.go b/examples/s3/bucketexists.go index e4b0a3a0e..5f98e0838 100644 --- a/examples/s3/bucketexists.go +++ b/examples/s3/bucketexists.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -39,7 +40,7 @@ func main() { log.Fatalln(err) } - found, err := s3Client.BucketExists("my-bucketname") + found, err := s3Client.BucketExists(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } diff --git a/examples/s3/composeobject.go b/examples/s3/composeobject.go index 78bfe9500..c628c2f29 100644 --- a/examples/s3/composeobject.go +++ b/examples/s3/composeobject.go @@ -20,10 +20,11 @@ package main import ( + "context" "log" - minio "github.com/minio/minio-go/v6" - "github.com/minio/minio-go/v6/pkg/encrypt" + minio "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/encrypt" ) func main() { @@ -70,7 +71,7 @@ func main() { log.Fatalln(err) } - err = s3Client.ComposeObject(dst, srcs) + err = s3Client.ComposeObject(context.Background(), dst, srcs) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/copyobject-with-new-tags.go b/examples/s3/copyobject-with-new-tags.go index bf9a12d39..b0496d1cb 100644 --- a/examples/s3/copyobject-with-new-tags.go +++ b/examples/s3/copyobject-with-new-tags.go @@ -20,10 +20,11 @@ package main import ( + "context" "log" "time" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -73,7 +74,7 @@ func main() { log.Fatalln(err) } // Initiate copy object. - err = s3Client.CopyObject(dst, src) + err = s3Client.CopyObject(context.Background(), dst, src) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/copyobject.go b/examples/s3/copyobject.go index 1c3f65a3a..390bb5d99 100644 --- a/examples/s3/copyobject.go +++ b/examples/s3/copyobject.go @@ -20,10 +20,11 @@ package main import ( + "context" "log" "time" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -67,7 +68,7 @@ func main() { } // Initiate copy object. - err = s3Client.CopyObject(dst, src) + err = s3Client.CopyObject(context.Background(), dst, src) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/deletebucketencryption.go b/examples/s3/deletebucketencryption.go index f050bc230..7345fde8f 100644 --- a/examples/s3/deletebucketencryption.go +++ b/examples/s3/deletebucketencryption.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -42,7 +43,7 @@ func main() { // s3Client.TraceOn(os.Stderr) // Get default encryption configuration set on a S3 bucket - err = s3Client.DeleteBucketEncryption("my-bucketname") + err = s3Client.DeleteBucketEncryption(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } diff --git a/examples/s3/disableversioning.go b/examples/s3/disableversioning.go index bbaec5e0c..711d666e8 100644 --- a/examples/s3/disableversioning.go +++ b/examples/s3/disableversioning.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - minio "github.com/minio/minio-go/v6" + minio "github.com/minio/minio-go/v7" ) func main() { @@ -39,7 +40,7 @@ func main() { log.Fatalln(err) } - err = s3Client.DisableVersioning("my-bucketname") + err = s3Client.DisableVersioning(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } diff --git a/examples/s3/enableversioning.go b/examples/s3/enableversioning.go index 58179be36..4de839852 100644 --- a/examples/s3/enableversioning.go +++ b/examples/s3/enableversioning.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - minio "github.com/minio/minio-go/v6" + minio "github.com/minio/minio-go/v7" ) func main() { @@ -39,7 +40,7 @@ func main() { log.Fatalln(err) } - err = s3Client.EnableVersioning("my-bucketname") + err = s3Client.EnableVersioning(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } diff --git a/examples/s3/fgetobject-context.go b/examples/s3/fgetobject-context.go deleted file mode 100644 index f3dc74d7b..000000000 --- a/examples/s3/fgetobject-context.go +++ /dev/null @@ -1,54 +0,0 @@ -// +build ignore - -/* - * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "log" - "time" - - "context" - - "github.com/minio/minio-go/v6" -) - -func main() { - // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname - // and my-filename.csv are dummy values, please replace them with original values. - - // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. - // This boolean value is the last argument for New(). - - // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically - // determined based on the Endpoint value. - - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) - if err != nil { - log.Fatalln(err) - } - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer cancel() - - if err := s3Client.FGetObjectWithContext(ctx, "my-bucketname", "my-objectname", "my-filename.csv", minio.GetObjectOptions{}); err != nil { - log.Fatalln(err) - } - log.Println("Successfully saved my-filename.csv") - -} diff --git a/examples/s3/fgetobject.go b/examples/s3/fgetobject.go index 76dd59f3f..33305657c 100644 --- a/examples/s3/fgetobject.go +++ b/examples/s3/fgetobject.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -39,7 +40,7 @@ func main() { log.Fatalln(err) } - if err := s3Client.FGetObject("my-bucketname", "my-objectname", "my-filename.csv", minio.GetObjectOptions{}); err != nil { + if err := s3Client.FGetObject(context.Background(), "my-bucketname", "my-objectname", "my-filename.csv", minio.GetObjectOptions{}); err != nil { log.Fatalln(err) } log.Println("Successfully saved my-filename.csv") diff --git a/examples/s3/fputencrypted-object.go b/examples/s3/fputencrypted-object.go index 95eefeafd..1d03beca7 100644 --- a/examples/s3/fputencrypted-object.go +++ b/examples/s3/fputencrypted-object.go @@ -20,10 +20,11 @@ package main import ( + "context" "log" - "github.com/minio/minio-go/v6" - "github.com/minio/minio-go/v6/pkg/encrypt" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/encrypt" ) func main() { @@ -49,7 +50,7 @@ func main() { encryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketname+objectName)) // Encrypt file content and upload to the server - n, err := s3Client.FPutObject(bucketname, objectName, filePath, minio.PutObjectOptions{ServerSideEncryption: encryption}) + n, err := s3Client.FPutObject(context.Background(), bucketname, objectName, filePath, minio.PutObjectOptions{ServerSideEncryption: encryption}) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/fputobject-context.go b/examples/s3/fputobject-context.go deleted file mode 100644 index 0aa300c2d..000000000 --- a/examples/s3/fputobject-context.go +++ /dev/null @@ -1,53 +0,0 @@ -// +build ignore - -/* - * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "log" - "time" - - "context" - - "github.com/minio/minio-go/v6" -) - -func main() { - // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname - // and my-filename.csv are dummy values, please replace them with original values. - - // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. - // This boolean value is the last argument for New(). - - // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically - // determined based on the Endpoint value. - - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) - if err != nil { - log.Fatalln(err) - } - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer cancel() - - if _, err := s3Client.FPutObjectWithContext(ctx, "my-bucketname", "my-objectname", "my-filename.csv", minio.PutObjectOptions{ContentType: "application/csv"}); err != nil { - log.Fatalln(err) - } - log.Println("Successfully uploaded my-filename.csv") -} diff --git a/examples/s3/fputobject.go b/examples/s3/fputobject.go index 2fc359545..458c09c38 100644 --- a/examples/s3/fputobject.go +++ b/examples/s3/fputobject.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -39,7 +40,7 @@ func main() { log.Fatalln(err) } - if _, err := s3Client.FPutObject("my-bucketname", "my-objectname", "my-filename.csv", minio.PutObjectOptions{ + if _, err := s3Client.FPutObject(context.Background(), "my-bucketname", "my-objectname", "my-filename.csv", minio.PutObjectOptions{ ContentType: "application/csv", }); err != nil { log.Fatalln(err) diff --git a/examples/s3/get-encrypted-object.go b/examples/s3/get-encrypted-object.go index 260770daf..732c7ad9b 100644 --- a/examples/s3/get-encrypted-object.go +++ b/examples/s3/get-encrypted-object.go @@ -20,12 +20,13 @@ package main import ( + "context" "io" "log" "os" - "github.com/minio/minio-go/v6" - "github.com/minio/minio-go/v6/pkg/encrypt" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/encrypt" ) func main() { @@ -50,7 +51,7 @@ func main() { encryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketname+objectName)) // Get the encrypted object - reader, err := s3Client.GetObject(bucketname, objectName, minio.GetObjectOptions{ServerSideEncryption: encryption}) + reader, err := s3Client.GetObject(context.Background(), bucketname, objectName, minio.GetObjectOptions{ServerSideEncryption: encryption}) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getbucketencryption.go b/examples/s3/getbucketencryption.go index ef47d5797..719ead8ff 100644 --- a/examples/s3/getbucketencryption.go +++ b/examples/s3/getbucketencryption.go @@ -20,10 +20,11 @@ package main import ( + "context" "fmt" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -44,7 +45,7 @@ func main() { // Get default encryption configuration set on an S3 bucket, // and print out the encryption configuration. - encryptionConfig, err := s3Client.GetBucketEncryption("my-bucketname") + encryptionConfig, err := s3Client.GetBucketEncryption(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getbucketlifecycle.go b/examples/s3/getbucketlifecycle.go index 1982a7639..7d7d701d8 100644 --- a/examples/s3/getbucketlifecycle.go +++ b/examples/s3/getbucketlifecycle.go @@ -20,12 +20,13 @@ package main import ( + "context" "io" "log" "os" "strings" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -45,7 +46,7 @@ func main() { // s3Client.TraceOn(os.Stderr) // Get bucket lifecycle from S3 - lifecycle, err := s3Client.GetBucketLifecycle("my-bucketname") + lifecycle, err := s3Client.GetBucketLifecycle(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getbucketnotification.go b/examples/s3/getbucketnotification.go index 1d711fc68..75909afea 100644 --- a/examples/s3/getbucketnotification.go +++ b/examples/s3/getbucketnotification.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -41,7 +42,7 @@ func main() { // s3Client.TraceOn(os.Stderr) - notifications, err := s3Client.GetBucketNotification("my-bucketname") + notifications, err := s3Client.GetBucketNotification(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getbucketpolicy.go b/examples/s3/getbucketpolicy.go index b10e933ac..c11e838ab 100644 --- a/examples/s3/getbucketpolicy.go +++ b/examples/s3/getbucketpolicy.go @@ -20,12 +20,13 @@ package main import ( + "context" "io" "log" "os" "strings" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -44,7 +45,7 @@ func main() { // s3Client.TraceOn(os.Stderr) - policy, err := s3Client.GetBucketPolicy("my-bucketname") + policy, err := s3Client.GetBucketPolicy(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getbuckettagging.go b/examples/s3/getbuckettagging.go index 27d87092a..2e7939849 100644 --- a/examples/s3/getbuckettagging.go +++ b/examples/s3/getbuckettagging.go @@ -20,10 +20,11 @@ package main import ( + "context" "fmt" "log" - minio "github.com/minio/minio-go/v6" + minio "github.com/minio/minio-go/v7" ) func main() { @@ -39,7 +40,7 @@ func main() { if err != nil { log.Fatalln(err) } - tags, err := s3Client.GetBucketTagging("my-bucketname") + tags, err := s3Client.GetBucketTagging(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getbucketversioning.go b/examples/s3/getbucketversioning.go index daf89cb96..d30ed30e7 100644 --- a/examples/s3/getbucketversioning.go +++ b/examples/s3/getbucketversioning.go @@ -20,10 +20,11 @@ package main import ( + "context" "fmt" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -44,7 +45,7 @@ func main() { // Get versioning configuration set on an S3 bucket, // and print out the versioning configuration. - versioningConfig, err := s3Client.GetBucketVersioning("my-bucketname") + versioningConfig, err := s3Client.GetBucketVersioning(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getobject-client-encryption.go b/examples/s3/getobject-client-encryption.go index 33fb0d2ed..f910072e7 100644 --- a/examples/s3/getobject-client-encryption.go +++ b/examples/s3/getobject-client-encryption.go @@ -20,11 +20,12 @@ package main import ( + "context" "log" "os" "path" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" "github.com/minio/sio" "golang.org/x/crypto/argon2" ) @@ -43,7 +44,7 @@ func main() { log.Fatalln(err) } - obj, err := s3Client.GetObject("my-bucketname", "my-objectname", minio.GetObjectOptions{}) + obj, err := s3Client.GetObject(context.Background(), "my-bucketname", "my-objectname", minio.GetObjectOptions{}) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getobject-context.go b/examples/s3/getobject-context.go deleted file mode 100644 index 1764d8aed..000000000 --- a/examples/s3/getobject-context.go +++ /dev/null @@ -1,73 +0,0 @@ -// +build ignore - -/* - * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "io" - "log" - "os" - "time" - - "context" - - "github.com/minio/minio-go/v6" -) - -func main() { - // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and - // my-testfile are dummy values, please replace them with original values. - - // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. - // This boolean value is the last argument for New(). - - // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically - // determined based on the Endpoint value. - - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) - if err != nil { - log.Fatalln(err) - } - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer cancel() - - opts := minio.GetObjectOptions{} - opts.SetModified(time.Now().Round(10 * time.Minute)) // get object if was modified within the last 10 minutes - reader, err := s3Client.GetObjectWithContext(ctx, "my-bucketname", "my-objectname", opts) - if err != nil { - log.Fatalln(err) - } - defer reader.Close() - - localFile, err := os.Create("my-testfile") - if err != nil { - log.Fatalln(err) - } - defer localFile.Close() - - stat, err := reader.Stat() - if err != nil { - log.Fatalln(err) - } - - if _, err := io.CopyN(localFile, reader, stat.Size); err != nil { - log.Fatalln(err) - } -} diff --git a/examples/s3/getobject.go b/examples/s3/getobject.go index eab9fc5aa..0bdd4ab87 100644 --- a/examples/s3/getobject.go +++ b/examples/s3/getobject.go @@ -20,11 +20,12 @@ package main import ( + "context" "io" "log" "os" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -41,7 +42,7 @@ func main() { log.Fatalln(err) } - reader, err := s3Client.GetObject("my-bucketname", "my-objectname", minio.GetObjectOptions{}) + reader, err := s3Client.GetObject(context.Background(), "my-bucketname", "my-objectname", minio.GetObjectOptions{}) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getobjectacl.go b/examples/s3/getobjectacl.go index 3b02a918d..9262c0942 100644 --- a/examples/s3/getobjectacl.go +++ b/examples/s3/getobjectacl.go @@ -20,10 +20,11 @@ package main import ( + "context" "fmt" "log" - minio "github.com/minio/minio-go/v6" + minio "github.com/minio/minio-go/v7" ) func main() { @@ -40,7 +41,7 @@ func main() { log.Fatalln(err) } - objectInfo, err := s3Client.GetObjectACL("my-bucketname", "my-objectname") + objectInfo, err := s3Client.GetObjectACL(context.Background(), "my-bucketname", "my-objectname") if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getobjectlegalhold.go b/examples/s3/getobjectlegalhold.go index 1ac7b7bdb..41d763865 100644 --- a/examples/s3/getobjectlegalhold.go +++ b/examples/s3/getobjectlegalhold.go @@ -20,10 +20,11 @@ package main import ( + "context" "fmt" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -40,7 +41,7 @@ func main() { log.Fatalln(err) } opts := minio.GetObjectLegalHoldOptions{} - lh, err := s3Client.GetObjectLegalHold("my-bucket", "my-object", opts) + lh, err := s3Client.GetObjectLegalHold(context.Background(), "my-bucket", "my-object", opts) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getobjectlockconfig.go b/examples/s3/getobjectlockconfig.go index cfdced1a7..547a007a9 100644 --- a/examples/s3/getobjectlockconfig.go +++ b/examples/s3/getobjectlockconfig.go @@ -20,11 +20,11 @@ package main import ( + "context" "fmt" "log" - "os" - minio "github.com/minio/minio-go/v6" + minio "github.com/minio/minio-go/v7" ) func main() { @@ -44,7 +44,7 @@ func main() { // s3Client.TraceOn(os.Stderr) // Get object lock configuration. - enabled, mode, validity, unit, err := s3Client.GetObjectLockConfig("tbucket13a") + enabled, mode, validity, unit, err := s3Client.GetObjectLockConfig(context.Background(), "tbucket13a") if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getobjectretention.go b/examples/s3/getobjectretention.go index 527f8a2ec..43c6f351b 100644 --- a/examples/s3/getobjectretention.go +++ b/examples/s3/getobjectretention.go @@ -20,11 +20,10 @@ package main import ( - "io" + "context" "log" - "os" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -40,7 +39,7 @@ func main() { if err != nil { log.Fatalln(err) } - m, t, err = s3Client.GetObjectRetention("my-bucket", "my-object", "") + m, t, err := s3Client.GetObjectRetention(context.Background(), "my-bucket", "my-object", "") if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getobjecttagging.go b/examples/s3/getobjecttagging.go index fb4b38396..ffd754e23 100644 --- a/examples/s3/getobjecttagging.go +++ b/examples/s3/getobjecttagging.go @@ -20,10 +20,11 @@ package main import ( + "context" "fmt" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -39,7 +40,7 @@ func main() { if err != nil { log.Fatalln(err) } - tags, err := s3Client.GetObjectTagging("my-bucketname", "my-objectname") + tags, err := s3Client.GetObjectTagging(context.Background(), "my-bucketname", "my-objectname") if err != nil { log.Fatalln(err) } diff --git a/examples/s3/listbuckets.go b/examples/s3/listbuckets.go index 9b1e23e97..dd360e85d 100644 --- a/examples/s3/listbuckets.go +++ b/examples/s3/listbuckets.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -39,7 +40,7 @@ func main() { log.Fatalln(err) } - buckets, err := s3Client.ListBuckets() + buckets, err := s3Client.ListBuckets(context.Background()) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/listincompleteuploads.go b/examples/s3/listincompleteuploads.go index da9bf59a7..462dfefa2 100644 --- a/examples/s3/listincompleteuploads.go +++ b/examples/s3/listincompleteuploads.go @@ -20,10 +20,11 @@ package main import ( + "context" "fmt" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -47,7 +48,7 @@ func main() { defer close(doneCh) // List all multipart uploads from a bucket-name with a matching prefix. - for multipartObject := range s3Client.ListIncompleteUploads("my-bucketname", "my-prefixname", true, doneCh) { + for multipartObject := range s3Client.ListIncompleteUploads(context.Background(), "my-bucketname", "my-prefixname", true, doneCh) { if multipartObject.Err != nil { fmt.Println(multipartObject.Err) return diff --git a/examples/s3/listobjects-N.go b/examples/s3/listobjects-N.go index 12afbbd42..9e189bd11 100644 --- a/examples/s3/listobjects-N.go +++ b/examples/s3/listobjects-N.go @@ -20,9 +20,10 @@ package main import ( + "context" "fmt" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -49,7 +50,7 @@ func main() { defer close(doneCh) i := 1 - for object := range s3Client.ListObjects(bucket, prefix, recursive, doneCh) { + for object := range s3Client.ListObjects(context.Background(), bucket, prefix, recursive, doneCh) { if object.Err != nil { return nil, object.Err } diff --git a/examples/s3/listobjects.go b/examples/s3/listobjects.go index 34fed8445..7bcb04797 100644 --- a/examples/s3/listobjects.go +++ b/examples/s3/listobjects.go @@ -20,9 +20,10 @@ package main import ( + "context" "fmt" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -47,7 +48,7 @@ func main() { defer close(doneCh) // List all objects from a bucket-name with a matching prefix. - for object := range s3Client.ListObjects("my-bucketname", "my-prefixname", true, doneCh) { + for object := range s3Client.ListObjects(context.Background(), "my-bucketname", "my-prefixname", true, doneCh) { if object.Err != nil { fmt.Println(object.Err) return diff --git a/examples/s3/listobjectsV2.go b/examples/s3/listobjectsV2.go index 7443e8b1a..7d92d82f9 100644 --- a/examples/s3/listobjectsV2.go +++ b/examples/s3/listobjectsV2.go @@ -20,9 +20,10 @@ package main import ( + "context" "fmt" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -47,7 +48,7 @@ func main() { defer close(doneCh) // List all objects from a bucket-name with a matching prefix. - for object := range s3Client.ListObjectsV2("my-bucketname", "my-prefixname", true, doneCh) { + for object := range s3Client.ListObjectsV2(context.Background(), "my-bucketname", "my-prefixname", true, doneCh) { if object.Err != nil { fmt.Println(object.Err) return diff --git a/examples/s3/listobjectsV2WithMetadata.go b/examples/s3/listobjectsV2WithMetadata.go index 77eb50919..c7de17782 100644 --- a/examples/s3/listobjectsV2WithMetadata.go +++ b/examples/s3/listobjectsV2WithMetadata.go @@ -20,9 +20,10 @@ package main import ( + "context" "fmt" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -47,7 +48,7 @@ func main() { defer close(doneCh) // List all objects from a bucket-name with a matching prefix. - for object := range s3Client.ListObjectsV2WithMetadata("my-bucketname", "my-prefixname", true, doneCh) { + for object := range s3Client.ListObjectsV2WithMetadata(context.Background(), "my-bucketname", "my-prefixname", true, doneCh) { if object.Err != nil { fmt.Println(object.Err) return diff --git a/examples/s3/makebucket.go b/examples/s3/makebucket.go index 1e179957f..e64fa3f63 100644 --- a/examples/s3/makebucket.go +++ b/examples/s3/makebucket.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -39,7 +40,7 @@ func main() { log.Fatalln(err) } - err = s3Client.MakeBucket("my-bucketname", "us-east-1") + err = s3Client.MakeBucket(context.Background(), "my-bucketname", "us-east-1") if err != nil { log.Fatalln(err) } diff --git a/examples/s3/makebucketwithobjectlock.go b/examples/s3/makebucketwithobjectlock.go index af52dd02b..e1fdcdfd2 100644 --- a/examples/s3/makebucketwithobjectlock.go +++ b/examples/s3/makebucketwithobjectlock.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - minio "github.com/minio/minio-go/v6" + minio "github.com/minio/minio-go/v7" ) func main() { @@ -39,7 +40,7 @@ func main() { log.Fatalln(err) } - err = s3Client.MakeBucketWithObjectLock("my-bucketname", "us-east-1") + err = s3Client.MakeBucketWithObjectLock(context.Background(), "my-bucketname", "us-east-1") if err != nil { log.Fatalln(err) } diff --git a/examples/s3/presignedgetobject.go b/examples/s3/presignedgetobject.go index dc870a029..3a8545bd5 100644 --- a/examples/s3/presignedgetobject.go +++ b/examples/s3/presignedgetobject.go @@ -20,11 +20,12 @@ package main import ( + "context" "log" "net/url" "time" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -46,7 +47,7 @@ func main() { reqParams.Set("response-content-disposition", "attachment; filename=\"your-filename.txt\"") // Gernerate presigned get object url. - presignedURL, err := s3Client.PresignedGetObject("my-bucketname", "my-objectname", time.Duration(1000)*time.Second, reqParams) + presignedURL, err := s3Client.PresignedGetObject(context.Background(), "my-bucketname", "my-objectname", time.Duration(1000)*time.Second, reqParams) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/presignedheadobject.go b/examples/s3/presignedheadobject.go index 8cec788c9..dacd32ed4 100644 --- a/examples/s3/presignedheadobject.go +++ b/examples/s3/presignedheadobject.go @@ -20,11 +20,12 @@ package main import ( + "context" "log" "net/url" "time" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -46,7 +47,7 @@ func main() { reqParams.Set("response-content-disposition", "attachment; filename=\"your-filename.txt\"") // Gernerate presigned get object url. - presignedURL, err := s3Client.PresignedHeadObject("my-bucketname", "my-objectname", time.Duration(1000)*time.Second, reqParams) + presignedURL, err := s3Client.PresignedHeadObject(context.Background(), "my-bucketname", "my-objectname", time.Duration(1000)*time.Second, reqParams) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/presignedpostpolicy.go b/examples/s3/presignedpostpolicy.go index 25e0f74d7..99c223b77 100644 --- a/examples/s3/presignedpostpolicy.go +++ b/examples/s3/presignedpostpolicy.go @@ -20,11 +20,12 @@ package main import ( + "context" "fmt" "log" "time" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -47,7 +48,7 @@ func main() { // Expires in 10 days. policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) // Returns form data for POST form request. - url, formData, err := s3Client.PresignedPostPolicy(policy) + url, formData, err := s3Client.PresignedPostPolicy(context.Background(), policy) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/presignedputobject.go b/examples/s3/presignedputobject.go index d200faa5b..c8e2d1847 100644 --- a/examples/s3/presignedputobject.go +++ b/examples/s3/presignedputobject.go @@ -20,10 +20,11 @@ package main import ( + "context" "log" "time" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -40,7 +41,7 @@ func main() { log.Fatalln(err) } - presignedURL, err := s3Client.PresignedPutObject("my-bucketname", "my-objectname", time.Duration(1000)*time.Second) + presignedURL, err := s3Client.PresignedPutObject(context.Background(), "my-bucketname", "my-objectname", time.Duration(1000)*time.Second) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/put-encrypted-object.go b/examples/s3/put-encrypted-object.go index 7d4cde286..720c6c0ac 100644 --- a/examples/s3/put-encrypted-object.go +++ b/examples/s3/put-encrypted-object.go @@ -20,11 +20,12 @@ package main import ( + "context" "log" "os" - "github.com/minio/minio-go/v6" - "github.com/minio/minio-go/v6/pkg/encrypt" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/encrypt" ) func main() { @@ -64,7 +65,7 @@ func main() { encryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketname+objectName)) // Encrypt file content and upload to the server - n, err := s3Client.PutObject(bucketname, objectName, file, fstat.Size(), minio.PutObjectOptions{ServerSideEncryption: encryption}) + n, err := s3Client.PutObject(context.Background(), bucketname, objectName, file, fstat.Size(), minio.PutObjectOptions{ServerSideEncryption: encryption}) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/putobject-client-encryption.go b/examples/s3/putobject-client-encryption.go index 0c0d7bb2a..a033b3df4 100644 --- a/examples/s3/putobject-client-encryption.go +++ b/examples/s3/putobject-client-encryption.go @@ -20,11 +20,12 @@ package main import ( + "context" "log" "os" "path" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" "github.com/minio/sio" "golang.org/x/crypto/argon2" ) @@ -67,7 +68,7 @@ func main() { if err != nil { log.Fatalln(err) } - _, err = s3Client.PutObject("my-bucketname", "my-objectname", encrypted, int64(encSize), minio.PutObjectOptions{}) + _, err = s3Client.PutObject(context.Background(), "my-bucketname", "my-objectname", encrypted, int64(encSize), minio.PutObjectOptions{}) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/putobject-context.go b/examples/s3/putobject-context.go deleted file mode 100644 index 6a476d49e..000000000 --- a/examples/s3/putobject-context.go +++ /dev/null @@ -1,68 +0,0 @@ -// +build ignore - -/* - * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "log" - "os" - "time" - - "context" - - "github.com/minio/minio-go/v6" -) - -func main() { - // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and - // my-objectname are dummy values, please replace them with original values. - - // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. - // This boolean value is the last argument for New(). - - // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically - // determined based on the Endpoint value. - - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) - if err != nil { - log.Fatalln(err) - } - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer cancel() - - object, err := os.Open("my-testfile") - if err != nil { - log.Fatalln(err) - } - defer object.Close() - - objectStat, err := object.Stat() - if err != nil { - log.Fatalln(err) - } - - n, err := s3Client.PutObjectWithContext(ctx, "my-bucketname", "my-objectname", object, objectStat.Size(), minio.PutObjectOptions{ - ContentType: "application/octet-stream", - }) - if err != nil { - log.Fatalln(err) - } - log.Println("Uploaded", "my-objectname", " of size: ", n, "Successfully.") -} diff --git a/examples/s3/putobject-getobject-sse.go b/examples/s3/putobject-getobject-sse.go index 110c323a2..4c54f2c0a 100644 --- a/examples/s3/putobject-getobject-sse.go +++ b/examples/s3/putobject-getobject-sse.go @@ -21,11 +21,12 @@ package main import ( "bytes" + "context" "io/ioutil" "log" - "github.com/minio/minio-go/v6" - "github.com/minio/minio-go/v6/pkg/encrypt" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/encrypt" ) func main() { @@ -44,14 +45,14 @@ func main() { object := []byte("Hello again") encryption := encrypt.DefaultPBKDF([]byte("my secret password"), []byte(bucketName+objectName)) - _, err = minioClient.PutObject(bucketName, objectName, bytes.NewReader(object), int64(len(object)), minio.PutObjectOptions{ + _, err = minioClient.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(object), int64(len(object)), minio.PutObjectOptions{ ServerSideEncryption: encryption, }) if err != nil { log.Fatalln(err) } - reader, err := minioClient.GetObject(bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: encryption}) + reader, err := minioClient.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: encryption}) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/putobject-progress.go b/examples/s3/putobject-progress.go index 54ce7cbfb..361e21e8d 100644 --- a/examples/s3/putobject-progress.go +++ b/examples/s3/putobject-progress.go @@ -20,10 +20,11 @@ package main import ( + "context" "log" "github.com/cheggaaa/pb" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -40,7 +41,7 @@ func main() { log.Fatalln(err) } - reader, err := s3Client.GetObject("my-bucketname", "my-objectname", minio.GetObjectOptions{}) + reader, err := s3Client.GetObject(context.Background(), "my-bucketname", "my-objectname", minio.GetObjectOptions{}) if err != nil { log.Fatalln(err) } @@ -55,7 +56,7 @@ func main() { // the Reads inside. progress := pb.New64(objectInfo.Size) progress.Start() - n, err := s3Client.PutObject("my-bucketname", "my-objectname-progress", reader, objectInfo.Size, minio.PutObjectOptions{ContentType: "application/octet-stream", Progress: progress}) + n, err := s3Client.PutObject(context.Background(), "my-bucketname", "my-objectname-progress", reader, objectInfo.Size, minio.PutObjectOptions{ContentType: "application/octet-stream", Progress: progress}) if err != nil { log.Fatalln(err) diff --git a/examples/s3/putobject-s3-accelerate.go b/examples/s3/putobject-s3-accelerate.go index d36af0a06..cc738145f 100644 --- a/examples/s3/putobject-s3-accelerate.go +++ b/examples/s3/putobject-s3-accelerate.go @@ -20,10 +20,11 @@ package main import ( + "context" "log" "os" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -54,7 +55,7 @@ func main() { log.Fatalln(err) } - n, err := s3Client.PutObject("my-bucketname", "my-objectname", object, objectStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) + n, err := s3Client.PutObject(context.Background(), "my-bucketname", "my-objectname", object, objectStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/putobject-streaming.go b/examples/s3/putobject-streaming.go index a90e4720f..3b14b1598 100644 --- a/examples/s3/putobject-streaming.go +++ b/examples/s3/putobject-streaming.go @@ -20,10 +20,11 @@ package main import ( + "context" "log" "os" - minio "github.com/minio/minio-go/v6" + minio "github.com/minio/minio-go/v7" ) func main() { @@ -46,7 +47,7 @@ func main() { } defer object.Close() - n, err := s3Client.PutObject("my-bucketname", "my-objectname", object, -1, minio.PutObjectOptions{}) + n, err := s3Client.PutObject(context.Background(), "my-bucketname", "my-objectname", object, -1, minio.PutObjectOptions{}) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/putobject-with-tags.go b/examples/s3/putobject-with-tags.go index 5c7ae6bea..6818933ce 100644 --- a/examples/s3/putobject-with-tags.go +++ b/examples/s3/putobject-with-tags.go @@ -20,10 +20,11 @@ package main import ( + "context" "log" "os" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -53,7 +54,7 @@ func main() { "Tag1": "Value1", "Tag2": "Value2", } - n, err := s3Client.PutObject("my-bucketname", "my-objectname", object, objectStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream", UserTags: tags}) + n, err := s3Client.PutObject(context.Background(), "my-bucketname", "my-objectname", object, objectStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream", UserTags: tags}) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/putobject.go b/examples/s3/putobject.go index 31969e035..43251c23d 100644 --- a/examples/s3/putobject.go +++ b/examples/s3/putobject.go @@ -20,10 +20,11 @@ package main import ( + "context" "log" "os" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -50,7 +51,7 @@ func main() { log.Fatalln(err) } - n, err := s3Client.PutObject("my-bucketname", "my-objectname", object, objectStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) + n, err := s3Client.PutObject(context.Background(), "my-bucketname", "my-objectname", object, objectStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/putobjectlegalhold.go b/examples/s3/putobjectlegalhold.go index 93cd43586..9d9842fed 100644 --- a/examples/s3/putobjectlegalhold.go +++ b/examples/s3/putobjectlegalhold.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -42,7 +43,7 @@ func main() { opts := minio.PutObjectLegalHoldOptions{ Status: &s, } - err = s3Client.PutObjectLegalHold("my-bucket", "my-object", opts) + err = s3Client.PutObjectLegalHold(context.Background(), "my-bucket", "my-object", opts) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/putobjectretention.go b/examples/s3/putobjectretention.go index b0db21a5b..3b2be844e 100644 --- a/examples/s3/putobjectretention.go +++ b/examples/s3/putobjectretention.go @@ -20,10 +20,11 @@ package main import ( + "context" "log" "time" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -46,7 +47,7 @@ func main() { RetainUntilDate: &t, Mode: &m, } - err = s3Client.PutObjectRetention("my-bucket", "my-object", opts) + err = s3Client.PutObjectRetention(context.Background(), "my-bucket", "my-object", opts) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/putobjecttagging.go b/examples/s3/putobjecttagging.go index a7532e694..32fcbd248 100644 --- a/examples/s3/putobjecttagging.go +++ b/examples/s3/putobjecttagging.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -42,7 +43,7 @@ func main() { "Tag1": "Value1", "Tag2": "Value2", } - err = s3Client.PutObjectTagging("my-bucketname", "my-objectname", tags) + err = s3Client.PutObjectTagging(context.Background(), "my-bucketname", "my-objectname", tags) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/removeallbucketnotification.go b/examples/s3/removeallbucketnotification.go index afd8ce13b..39a9eafd5 100644 --- a/examples/s3/removeallbucketnotification.go +++ b/examples/s3/removeallbucketnotification.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -41,7 +42,7 @@ func main() { // s3Client.TraceOn(os.Stderr) - err = s3Client.RemoveAllBucketNotification("my-bucketname") + err = s3Client.RemoveAllBucketNotification(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } diff --git a/examples/s3/removebucket.go b/examples/s3/removebucket.go index 7f2295e6a..12f918ce9 100644 --- a/examples/s3/removebucket.go +++ b/examples/s3/removebucket.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -40,7 +41,7 @@ func main() { } // This operation will only work if your bucket is empty. - err = s3Client.RemoveBucket("my-bucketname") + err = s3Client.RemoveBucket(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } diff --git a/examples/s3/removebuckettagging.go b/examples/s3/removebuckettagging.go index 76780aa35..b78528876 100644 --- a/examples/s3/removebuckettagging.go +++ b/examples/s3/removebuckettagging.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - minio "github.com/minio/minio-go/v6" + minio "github.com/minio/minio-go/v7" ) func main() { @@ -38,7 +39,7 @@ func main() { if err != nil { log.Fatalln(err) } - err = s3Client.DeleteBucketTagging("my-bucketname") + err = s3Client.DeleteBucketTagging(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } diff --git a/examples/s3/removeincompleteupload.go b/examples/s3/removeincompleteupload.go index 9b941ea62..1681b016b 100644 --- a/examples/s3/removeincompleteupload.go +++ b/examples/s3/removeincompleteupload.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -39,7 +40,7 @@ func main() { log.Fatalln(err) } - err = s3Client.RemoveIncompleteUpload("my-bucketname", "my-objectname") + err = s3Client.RemoveIncompleteUpload(context.Background(), "my-bucketname", "my-objectname") if err != nil { log.Fatalln(err) } diff --git a/examples/s3/removeobject.go b/examples/s3/removeobject.go index 967ec8c5f..547b5aa4e 100644 --- a/examples/s3/removeobject.go +++ b/examples/s3/removeobject.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -38,7 +39,7 @@ func main() { if err != nil { log.Fatalln(err) } - err = s3Client.RemoveObject("my-bucketname", "my-objectname") + err = s3Client.RemoveObject(context.Background(), "my-bucketname", "my-objectname") if err != nil { log.Fatalln(err) } diff --git a/examples/s3/removeobjectoptions.go b/examples/s3/removeobjectoptions.go index 926d80eed..edc408468 100644 --- a/examples/s3/removeobjectoptions.go +++ b/examples/s3/removeobjectoptions.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -41,7 +42,7 @@ func main() { opts := minio.RemoveObjectOptions{ GovernanceBypass: true, } - err = s3Client.RemoveObjectWithOptions("my-bucket", "my-object", opts) + err = s3Client.RemoveObjectWithOptions(context.Background(), "my-bucket", "my-object", opts) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/removeobjects.go b/examples/s3/removeobjects.go index d453beea2..681b0269d 100644 --- a/examples/s3/removeobjects.go +++ b/examples/s3/removeobjects.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -51,7 +52,7 @@ func main() { defer close(doneCh) // List all objects from a bucket-name with a matching prefix. - for object := range s3Client.ListObjects("my-bucketname", "my-prefixname", true, doneCh) { + for object := range s3Client.ListObjects(context.Background(), "my-bucketname", "my-prefixname", true, doneCh) { if object.Err != nil { log.Fatalln(object.Err) } @@ -60,7 +61,7 @@ func main() { }() // Call RemoveObjects API - errorCh := s3Client.RemoveObjects("my-bucketname", objectsCh) + errorCh := s3Client.RemoveObjects(context.Background(), "my-bucketname", objectsCh) // Print errors received from RemoveObjects API for e := range errorCh { diff --git a/examples/s3/removeobjecttagging.go b/examples/s3/removeobjecttagging.go index 3fbae638a..47ac71d55 100644 --- a/examples/s3/removeobjecttagging.go +++ b/examples/s3/removeobjecttagging.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -38,7 +39,7 @@ func main() { if err != nil { log.Fatalln(err) } - err = s3Client.RemoveObjectTagging("my-bucketname", "my-objectname") + err = s3Client.RemoveObjectTagging(context.Background(), "my-bucketname", "my-objectname") if err != nil { log.Fatalln(err) } diff --git a/examples/s3/selectobject.go b/examples/s3/selectobject.go index e7a837036..249f34dd4 100644 --- a/examples/s3/selectobject.go +++ b/examples/s3/selectobject.go @@ -25,7 +25,7 @@ import ( "log" "os" - minio "github.com/minio/minio-go/v6" + minio "github.com/minio/minio-go/v7" ) func main() { diff --git a/examples/s3/setbucketencryption.go b/examples/s3/setbucketencryption.go index fda57c65d..e3e39f8b7 100644 --- a/examples/s3/setbucketencryption.go +++ b/examples/s3/setbucketencryption.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -51,7 +52,7 @@ func main() { }, }, }} - err = s3Client.SetBucketEncryption("my-bucketname", config) + err = s3Client.SetBucketEncryption(context.Background(), "my-bucketname", config) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/setbucketlifecycle.go b/examples/s3/setbucketlifecycle.go index 362caa8c9..927a060df 100644 --- a/examples/s3/setbucketlifecycle.go +++ b/examples/s3/setbucketlifecycle.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -43,7 +44,7 @@ func main() { // Set lifecycle on a bucket lifecycle := `expire-bucketEnabled365` - err = s3Client.SetBucketLifecycle("my-bucketname", lifecycle) + err = s3Client.SetBucketLifecycle(context.Background(), "my-bucketname", lifecycle) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/setbucketnotification.go b/examples/s3/setbucketnotification.go index bb6ab5e76..5e3b434d3 100644 --- a/examples/s3/setbucketnotification.go +++ b/examples/s3/setbucketnotification.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -78,7 +79,7 @@ func main() { bucketNotification.AddQueue(queueConfig) bucketNotification.AddLambda(lambdaConfig) - err = s3Client.SetBucketNotification("YOUR-BUCKET", bucketNotification) + err = s3Client.SetBucketNotification(context.Background(), "YOUR-BUCKET", bucketNotification) if err != nil { log.Fatalln("Error: " + err.Error()) } diff --git a/examples/s3/setbucketpolicy.go b/examples/s3/setbucketpolicy.go index 607627865..8166a1e3b 100644 --- a/examples/s3/setbucketpolicy.go +++ b/examples/s3/setbucketpolicy.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -44,7 +45,7 @@ func main() { // Create policy policy := `{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::my-bucketname/*"],"Sid": ""}]}` - err = s3Client.SetBucketPolicy("my-bucketname", policy) + err = s3Client.SetBucketPolicy(context.Background(), "my-bucketname", policy) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/setbuckettagging.go b/examples/s3/setbuckettagging.go index 3f6f9c856..45efc28b2 100644 --- a/examples/s3/setbuckettagging.go +++ b/examples/s3/setbuckettagging.go @@ -20,10 +20,11 @@ package main import ( + "context" "log" - minio "github.com/minio/minio-go/v6" - "github.com/minio/minio-go/v6/pkg/tags" + minio "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/tags" ) func main() { @@ -48,7 +49,7 @@ func main() { log.Fatalln(err) } - err = s3Client.SetBucketTagging("my-bucketname", tags) + err = s3Client.SetBucketTagging(context.Background(), "my-bucketname", tags) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/setobjectlockconfig.go b/examples/s3/setobjectlockconfig.go index 6169f1770..e2ead60f2 100644 --- a/examples/s3/setobjectlockconfig.go +++ b/examples/s3/setobjectlockconfig.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - minio "github.com/minio/minio-go/v6" + minio "github.com/minio/minio-go/v7" ) func main() { @@ -46,7 +47,7 @@ func main() { validity := uint(30) unit := minio.Days - err = s3Client.SetObjectLockConfig("my-bucketname", &mode, &validity, &unit) + err = s3Client.SetObjectLockConfig(context.Background(), "my-bucketname", &mode, &validity, &unit) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/statobject.go b/examples/s3/statobject.go index 9e8fec65d..df2f1425e 100644 --- a/examples/s3/statobject.go +++ b/examples/s3/statobject.go @@ -20,9 +20,10 @@ package main import ( + "context" "log" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" ) func main() { @@ -38,7 +39,7 @@ func main() { if err != nil { log.Fatalln(err) } - stat, err := s3Client.StatObject("my-bucketname", "my-objectname", minio.StatObjectOptions{}) + stat, err := s3Client.StatObject(context.Background(), "my-bucketname", "my-objectname", minio.StatObjectOptions{}) if err != nil { log.Fatalln(err) } diff --git a/functional_tests.go b/functional_tests.go index dcdf30364..cf084ce3f 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -2,7 +2,7 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,8 +42,8 @@ import ( jsoniter "github.com/json-iterator/go" log "github.com/sirupsen/logrus" - "github.com/minio/minio-go/v6" - "github.com/minio/minio-go/v6/pkg/encrypt" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/encrypt" ) const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" @@ -148,30 +148,30 @@ func cleanupBucket(bucketName string, c *minio.Client) error { // Exit cleanly upon return. defer close(doneCh) // Iterate over all objects in the bucket via listObjectsV2 and delete - for objCh := range c.ListObjectsV2(bucketName, "", true, doneCh) { + for objCh := range c.ListObjectsV2(context.Background(), bucketName, "", true, doneCh) { if objCh.Err != nil { return objCh.Err } if objCh.Key != "" { - err := c.RemoveObject(bucketName, objCh.Key) + err := c.RemoveObject(context.Background(), bucketName, objCh.Key) if err != nil { return err } } } - for objPartInfo := range c.ListIncompleteUploads(bucketName, "", true, doneCh) { + for objPartInfo := range c.ListIncompleteUploads(context.Background(), bucketName, "", true, doneCh) { if objPartInfo.Err != nil { return objPartInfo.Err } if objPartInfo.Key != "" { - err := c.RemoveIncompleteUpload(bucketName, objPartInfo.Key) + err := c.RemoveIncompleteUpload(context.Background(), bucketName, objPartInfo.Key) if err != nil { return err } } } // objects are already deleted, clear the buckets now - err := c.RemoveBucket(bucketName) + err := c.RemoveBucket(context.Background(), bucketName) if err != nil { return err } @@ -322,11 +322,11 @@ func testMakeBucketError() { args["bucketName"] = bucketName // Make a new bucket in 'eu-central-1'. - if err = c.MakeBucket(bucketName, region); err != nil { + if err = c.MakeBucket(context.Background(), bucketName, region); err != nil { logError(testName, function, args, startTime, "", "MakeBucket Failed", err) return } - if err = c.MakeBucket(bucketName, region); err == nil { + if err = c.MakeBucket(context.Background(), bucketName, region); err == nil { logError(testName, function, args, startTime, "", "Bucket already exists", err) return } @@ -374,7 +374,7 @@ func testMetadataSizeLimit() { objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return @@ -388,7 +388,7 @@ func testMetadataSizeLimit() { metadata["X-Amz-Meta-Mint-Test"] = string(bytes.Repeat([]byte("m"), 1+UserMetadataLimit-len("X-Amz-Meta-Mint-Test"))) args["metadata"] = fmt.Sprint(metadata) - _, err = c.PutObject(bucketName, objectName, bytes.NewReader(nil), 0, minio.PutObjectOptions{UserMetadata: metadata}) + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(nil), 0, minio.PutObjectOptions{UserMetadata: metadata}) if err == nil { logError(testName, function, args, startTime, "", "Created object with user-defined metadata exceeding metadata size limits", nil) return @@ -398,7 +398,7 @@ func testMetadataSizeLimit() { metadata = make(map[string]string) metadata["X-Amz-Mint-Test"] = string(bytes.Repeat([]byte("m"), 1+HeaderSizeLimit-len("X-Amz-Mint-Test"))) args["metadata"] = fmt.Sprint(metadata) - _, err = c.PutObject(bucketName, objectName, bytes.NewReader(nil), 0, minio.PutObjectOptions{UserMetadata: metadata}) + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(nil), 0, minio.PutObjectOptions{UserMetadata: metadata}) if err == nil { logError(testName, function, args, startTime, "", "Created object with headers exceeding header size limits", nil) return @@ -458,7 +458,7 @@ func testMakeBucketRegions() { args["bucketName"] = bucketName // Make a new bucket in 'eu-central-1'. - if err = c.MakeBucket(bucketName, region); err != nil { + if err = c.MakeBucket(context.Background(), bucketName, region); err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } @@ -474,7 +474,7 @@ func testMakeBucketRegions() { // virtual host style. region = "us-west-2" args["region"] = region - if err = c.MakeBucket(bucketName+".withperiod", region); err != nil { + if err = c.MakeBucket(context.Background(), bucketName+".withperiod", region); err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } @@ -525,7 +525,7 @@ func testPutObjectReadAt() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return @@ -543,7 +543,7 @@ func testPutObjectReadAt() { objectContentType := "binary/octet-stream" args["objectContentType"] = objectContentType - n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: objectContentType}) + n, err := c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: objectContentType}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -555,7 +555,7 @@ func testPutObjectReadAt() { } // Read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "Get Object failed", err) return @@ -635,7 +635,7 @@ func testPutObjectWithMetadata() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return @@ -657,7 +657,7 @@ func testPutObjectWithMetadata() { "X-Amz-Meta-CustomKey": {"extra spaces in value"}, } - n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ + n, err := c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ ContentType: customContentType}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) @@ -670,7 +670,7 @@ func testPutObjectWithMetadata() { } // Read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return @@ -745,14 +745,14 @@ func testPutObjectWithContentLanguage() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } data := bytes.Repeat([]byte("a"), int(0)) - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(data), int64(0), minio.PutObjectOptions{ + n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(data), int64(0), minio.PutObjectOptions{ ContentLanguage: "en", }) if err != nil { @@ -765,7 +765,7 @@ func testPutObjectWithContentLanguage() { return } - objInfo, err := c.StatObject(bucketName, objectName, minio.StatObjectOptions{}) + objInfo, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return @@ -824,7 +824,7 @@ func testPutObjectStreaming() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -835,7 +835,7 @@ func testPutObjectStreaming() { for _, size := range sizes { data := bytes.Repeat([]byte("a"), int(size)) - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(data), int64(size), minio.PutObjectOptions{}) + n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(data), int64(size), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectStreaming failed", err) return @@ -890,7 +890,7 @@ func testGetObjectSeekEnd() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -911,7 +911,7 @@ func testGetObjectSeekEnd() { return } - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -923,7 +923,7 @@ func testGetObjectSeekEnd() { } // Read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return @@ -1022,7 +1022,7 @@ func testGetObjectClosedTwice() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -1037,7 +1037,7 @@ func testGetObjectClosedTwice() { objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName - n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + n, err := c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -1049,7 +1049,7 @@ func testGetObjectClosedTwice() { } // Read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return @@ -1082,12 +1082,12 @@ func testGetObjectClosedTwice() { successLogger(testName, function, args, startTime).Info() } -// Test RemoveObjectsWithContext request context cancels after timeout +// Test RemoveObjects request where context cancels after timeout func testRemoveObjectsWithContext() { // Initialize logging params. startTime := time.Now() testName := getFuncName() - function := "RemoveObjectsWithContext(ctx, bucketName, objectsCh)" + function := "RemoveObjects(ctx, bucketName, objectsCh)" args := map[string]interface{}{ "bucketName": "", } @@ -1117,7 +1117,7 @@ func testRemoveObjectsWithContext() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) } @@ -1132,7 +1132,7 @@ func testRemoveObjectsWithContext() { defer close(objectsCh) for i := 0; i < nrObjects; i++ { objectName := "sample" + strconv.Itoa(i) + ".txt" - _, err = c.PutObject(bucketName, objectName, r, 8, minio.PutObjectOptions{ContentType: "application/octet-stream"}) + _, err = c.PutObject(context.Background(), bucketName, objectName, r, 8, minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) continue @@ -1145,13 +1145,13 @@ func testRemoveObjectsWithContext() { args["ctx"] = ctx defer cancel() - // Call RemoveObjectsWithContext API with short timeout. - errorCh := c.RemoveObjectsWithContext(ctx, bucketName, objectsCh) + // Call RemoveObjects API with short timeout. + errorCh := c.RemoveObjects(ctx, bucketName, objectsCh) // Check for error. select { case r := <-errorCh: if r.Err == nil { - logError(testName, function, args, startTime, "", "RemoveObjectsWithContext should fail on short timeout", err) + logError(testName, function, args, startTime, "", "RemoveObjects should fail on short timeout", err) return } } @@ -1159,8 +1159,8 @@ func testRemoveObjectsWithContext() { ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) args["ctx"] = ctx defer cancel() - // Perform RemoveObjectsWithContext with the longer timeout. Expect the removals to succeed. - errorCh = c.RemoveObjectsWithContext(ctx, bucketName, objectsCh) + // Perform RemoveObjects with the longer timeout. Expect the removals to succeed. + errorCh = c.RemoveObjects(ctx, bucketName, objectsCh) select { case r, more := <-errorCh: if more || r.Err != nil { @@ -1214,7 +1214,7 @@ func testRemoveMultipleObjects() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -1232,7 +1232,7 @@ func testRemoveMultipleObjects() { // Upload objects and send them to objectsCh for i := 0; i < nrObjects; i++ { objectName := "sample" + strconv.Itoa(i) + ".txt" - _, err = c.PutObject(bucketName, objectName, r, 8, minio.PutObjectOptions{ContentType: "application/octet-stream"}) + _, err = c.PutObject(context.Background(), bucketName, objectName, r, 8, minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) continue @@ -1242,7 +1242,7 @@ func testRemoveMultipleObjects() { }() // Call RemoveObjects API - errorCh := c.RemoveObjects(bucketName, objectsCh) + errorCh := c.RemoveObjects(context.Background(), bucketName, objectsCh) // Check if errorCh doesn't receive any error select { @@ -1301,7 +1301,7 @@ func testFPutObjectMultipart() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -1337,7 +1337,7 @@ func testFPutObjectMultipart() { args["objectContentType"] = objectContentType // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) - n, err := c.FPutObject(bucketName, objectName, fileName, minio.PutObjectOptions{ContentType: objectContentType}) + n, err := c.FPutObject(context.Background(), bucketName, objectName, fileName, minio.PutObjectOptions{ContentType: objectContentType}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject failed", err) return @@ -1347,7 +1347,7 @@ func testFPutObjectMultipart() { return } - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return @@ -1417,8 +1417,8 @@ func testFPutObject() { // Make a new bucket. args["bucketName"] = bucketName args["location"] = location - function = "MakeBucket()bucketName, location" - err = c.MakeBucket(bucketName, location) + function = "MakeBucket(bucketName, location)" + err = c.MakeBucket(context.Background(), bucketName, location) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -1458,7 +1458,7 @@ func testFPutObject() { args["opts"] = minio.PutObjectOptions{ContentType: "application/octet-stream"} // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) - n, err := c.FPutObject(bucketName, objectName+"-standard", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) + n, err := c.FPutObject(context.Background(), bucketName, objectName+"-standard", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject failed", err) @@ -1471,7 +1471,7 @@ func testFPutObject() { // Perform FPutObject with no contentType provided (Expecting application/octet-stream) args["objectName"] = objectName + "-Octet" - n, err = c.FPutObject(bucketName, objectName+"-Octet", fName, minio.PutObjectOptions{}) + n, err = c.FPutObject(context.Background(), bucketName, objectName+"-Octet", fName, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "File close failed", err) return @@ -1502,7 +1502,7 @@ func testFPutObject() { // Perform FPutObject with no contentType provided (Expecting application/x-gtar) args["objectName"] = objectName + "-GTar" args["opts"] = minio.PutObjectOptions{} - n, err = c.FPutObject(bucketName, objectName+"-GTar", fName+".gtar", minio.PutObjectOptions{}) + n, err = c.FPutObject(context.Background(), bucketName, objectName+"-GTar", fName+".gtar", minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject failed", err) return @@ -1515,7 +1515,7 @@ func testFPutObject() { // Check headers function = "StatObject(bucketName, objectName, opts)" args["objectName"] = objectName + "-standard" - rStandard, err := c.StatObject(bucketName, objectName+"-standard", minio.StatObjectOptions{}) + rStandard, err := c.StatObject(context.Background(), bucketName, objectName+"-standard", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return @@ -1527,7 +1527,7 @@ func testFPutObject() { function = "StatObject(bucketName, objectName, opts)" args["objectName"] = objectName + "-Octet" - rOctet, err := c.StatObject(bucketName, objectName+"-Octet", minio.StatObjectOptions{}) + rOctet, err := c.StatObject(context.Background(), bucketName, objectName+"-Octet", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return @@ -1539,7 +1539,7 @@ func testFPutObject() { function = "StatObject(bucketName, objectName, opts)" args["objectName"] = objectName + "-GTar" - rGTar, err := c.StatObject(bucketName, objectName+"-GTar", minio.StatObjectOptions{}) + rGTar, err := c.StatObject(context.Background(), bucketName, objectName+"-GTar", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return @@ -1559,7 +1559,7 @@ func testFPutObject() { successLogger(testName, function, args, startTime).Info() } -// Tests FPutObjectWithContext request context cancels after timeout +// Tests FPutObject request when context cancels after timeout func testFPutObjectWithContext() { // initialize logging params startTime := time.Now() @@ -1597,7 +1597,7 @@ func testFPutObjectWithContext() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -1636,18 +1636,18 @@ func testFPutObjectWithContext() { args["ctx"] = ctx defer cancel() - // Perform standard FPutObjectWithContext with contentType provided (Expecting application/octet-stream) - _, err = c.FPutObjectWithContext(ctx, bucketName, objectName+"-Shorttimeout", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) + // Perform FPutObject with contentType provided (Expecting application/octet-stream) + _, err = c.FPutObject(ctx, bucketName, objectName+"-Shorttimeout", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err == nil { - logError(testName, function, args, startTime, "", "FPutObjectWithContext should fail on short timeout", err) + logError(testName, function, args, startTime, "", "FPutObject should fail on short timeout", err) return } ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) defer cancel() - // Perform FPutObjectWithContext with a long timeout. Expect the put object to succeed - n, err := c.FPutObjectWithContext(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{}) + // Perform FPutObject with a long timeout. Expect the put object to succeed + n, err := c.FPutObject(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{}) if err != nil { - logError(testName, function, args, startTime, "", "FPutObjectWithContext shouldn't fail on long timeout", err) + logError(testName, function, args, startTime, "", "FPutObject shouldn't fail on long timeout", err) return } if n != int64(totalSize) { @@ -1655,7 +1655,7 @@ func testFPutObjectWithContext() { return } - _, err = c.StatObject(bucketName, objectName+"-Longtimeout", minio.StatObjectOptions{}) + _, err = c.StatObject(context.Background(), bucketName, objectName+"-Longtimeout", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return @@ -1671,7 +1671,7 @@ func testFPutObjectWithContext() { } -// Tests FPutObjectWithContext request context cancels after timeout +// Tests FPutObject request when context cancels after timeout func testFPutObjectWithContextV2() { // initialize logging params startTime := time.Now() @@ -1708,7 +1708,7 @@ func testFPutObjectWithContextV2() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -1749,18 +1749,18 @@ func testFPutObjectWithContextV2() { args["ctx"] = ctx defer cancel() - // Perform standard FPutObjectWithContext with contentType provided (Expecting application/octet-stream) - _, err = c.FPutObjectWithContext(ctx, bucketName, objectName+"-Shorttimeout", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) + // Perform FPutObject with contentType provided (Expecting application/octet-stream) + _, err = c.FPutObject(ctx, bucketName, objectName+"-Shorttimeout", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err == nil { - logError(testName, function, args, startTime, "", "FPutObjectWithContext should fail on short timeout", err) + logError(testName, function, args, startTime, "", "FPutObject should fail on short timeout", err) return } ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) defer cancel() - // Perform FPutObjectWithContext with a long timeout. Expect the put object to succeed - n, err := c.FPutObjectWithContext(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{}) + // Perform FPutObject with a long timeout. Expect the put object to succeed + n, err := c.FPutObject(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{}) if err != nil { - logError(testName, function, args, startTime, "", "FPutObjectWithContext shouldn't fail on longer timeout", err) + logError(testName, function, args, startTime, "", "FPutObject shouldn't fail on longer timeout", err) return } if n != int64(totalSize) { @@ -1768,7 +1768,7 @@ func testFPutObjectWithContextV2() { return } - _, err = c.StatObject(bucketName, objectName+"-Longtimeout", minio.StatObjectOptions{}) + _, err = c.StatObject(context.Background(), bucketName, objectName+"-Longtimeout", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return @@ -1789,7 +1789,7 @@ func testPutObjectWithContext() { // initialize logging params startTime := time.Now() testName := getFuncName() - function := "PutObjectWithContext(ctx, bucketName, objectName, fileName, opts)" + function := "PutObject(ctx, bucketName, objectName, fileName, opts)" args := map[string]interface{}{ "ctx": "", "bucketName": "", @@ -1818,7 +1818,7 @@ func testPutObjectWithContext() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket call failed", err) return @@ -1834,9 +1834,9 @@ func testPutObjectWithContext() { args["opts"] = minio.PutObjectOptions{ContentType: "binary/octet-stream"} defer cancel() - _, err = c.PutObjectWithContext(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + _, err = c.PutObject(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err == nil { - logError(testName, function, args, startTime, "", "PutObjectWithContext should fail on short timeout", err) + logError(testName, function, args, startTime, "", "PutObject should fail on short timeout", err) return } @@ -1846,9 +1846,9 @@ func testPutObjectWithContext() { defer cancel() reader = getDataReader("datafile-33-kB") defer reader.Close() - _, err = c.PutObjectWithContext(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + _, err = c.PutObject(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { - logError(testName, function, args, startTime, "", "PutObjectWithContext with long timeout failed", err) + logError(testName, function, args, startTime, "", "PutObject with long timeout failed", err) return } @@ -1896,7 +1896,7 @@ func testGetObjectReadSeekFunctional() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -1925,7 +1925,7 @@ func testGetObjectReadSeekFunctional() { } // Save the data - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -1937,7 +1937,7 @@ func testGetObjectReadSeekFunctional() { } // Read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return @@ -2071,7 +2071,7 @@ func testGetObjectReadAtFunctional() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -2092,7 +2092,7 @@ func testGetObjectReadAtFunctional() { } // Save the data - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -2104,7 +2104,7 @@ func testGetObjectReadAtFunctional() { } // read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -2256,7 +2256,7 @@ func testGetObjectReadAtWhenEOFWasReached() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -2277,7 +2277,7 @@ func testGetObjectReadAtWhenEOFWasReached() { } // Save the data - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -2289,7 +2289,7 @@ func testGetObjectReadAtWhenEOFWasReached() { } // read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -2384,7 +2384,7 @@ func testPresignedPostPolicy() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -2407,7 +2407,7 @@ func testPresignedPostPolicy() { } // Save the data - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -2457,7 +2457,7 @@ func testPresignedPostPolicy() { policy.SetUserMetadata(metadataKey, metadataValue) args["policy"] = policy.String() - presignedPostPolicyURL, formData, err := c.PresignedPostPolicy(policy) + presignedPostPolicyURL, formData, err := c.PresignedPostPolicy(context.Background(), policy) if err != nil { logError(testName, function, args, startTime, "", "PresignedPostPolicy failed", err) return @@ -2606,14 +2606,14 @@ func testCopyObject() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } // Make a new bucket in 'us-east-1' (destination bucket). - err = c.MakeBucket(bucketName+"-copy", "us-east-1") + err = c.MakeBucket(context.Background(), bucketName+"-copy", "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -2625,7 +2625,7 @@ func testCopyObject() { // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + n, err := c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -2636,7 +2636,7 @@ func testCopyObject() { return } - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return @@ -2695,21 +2695,21 @@ func testCopyObject() { } // Perform the Copy - err = c.CopyObject(dst, src) + err = c.CopyObject(context.Background(), dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } // Source object - r, err = c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + r, err = c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } // Destination object - readerCopy, err := c.GetObject(bucketName+"-copy", objectName+"-copy", minio.GetObjectOptions{}) + readerCopy, err := c.GetObject(context.Background(), bucketName+"-copy", objectName+"-copy", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return @@ -2748,7 +2748,7 @@ func testCopyObject() { } // Perform the Copy which should fail - err = c.CopyObject(dst, src) + err = c.CopyObject(context.Background(), dst, src) if err == nil { logError(testName, function, args, startTime, "", "CopyObject did not fail for invalid conditions", err) return @@ -2766,13 +2766,13 @@ func testCopyObject() { return } - err = c.CopyObject(dst, src) + err = c.CopyObject(context.Background(), dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject shouldn't fail", err) return } - oi, err := c.StatObject(bucketName, objectName, minio.StatObjectOptions{}) + oi, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return @@ -2780,7 +2780,7 @@ func testCopyObject() { stOpts := minio.StatObjectOptions{} stOpts.SetMatchETag(oi.ETag) - objInfo, err = c.StatObject(bucketName, objectName, stOpts) + objInfo, err = c.StatObject(context.Background(), bucketName, objectName, stOpts) if err != nil { logError(testName, function, args, startTime, "", "CopyObject ETag should match and not fail", err) return @@ -2837,7 +2837,7 @@ func testSSECEncryptedGetObjectReadSeekFunctional() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -2866,7 +2866,7 @@ func testSSECEncryptedGetObjectReadSeekFunctional() { } // Save the data - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ + n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ ContentType: "binary/octet-stream", ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), }) @@ -2881,7 +2881,7 @@ func testSSECEncryptedGetObjectReadSeekFunctional() { } // Read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{ + r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{ ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), }) if err != nil { @@ -3025,7 +3025,7 @@ func testSSES3EncryptedGetObjectReadSeekFunctional() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -3054,7 +3054,7 @@ func testSSES3EncryptedGetObjectReadSeekFunctional() { } // Save the data - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ + n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ ContentType: "binary/octet-stream", ServerSideEncryption: encrypt.NewSSE(), }) @@ -3069,7 +3069,7 @@ func testSSES3EncryptedGetObjectReadSeekFunctional() { } // Read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return @@ -3211,7 +3211,7 @@ func testSSECEncryptedGetObjectReadAtFunctional() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -3232,7 +3232,7 @@ func testSSECEncryptedGetObjectReadAtFunctional() { } // Save the data - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ + n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ ContentType: "binary/octet-stream", ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), }) @@ -3247,7 +3247,7 @@ func testSSECEncryptedGetObjectReadAtFunctional() { } // read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{ + r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{ ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), }) if err != nil { @@ -3402,7 +3402,7 @@ func testSSES3EncryptedGetObjectReadAtFunctional() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -3423,7 +3423,7 @@ func testSSES3EncryptedGetObjectReadAtFunctional() { } // Save the data - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ + n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ ContentType: "binary/octet-stream", ServerSideEncryption: encrypt.NewSSE(), }) @@ -3438,7 +3438,7 @@ func testSSES3EncryptedGetObjectReadAtFunctional() { } // read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -3594,7 +3594,7 @@ func testSSECEncryptionPutGet() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -3627,14 +3627,14 @@ func testSSECEncryptionPutGet() { args["sse"] = sse // Put encrypted data - _, err = c.PutObject(bucketName, objectName, bytes.NewReader(testCase.buf), int64(len(testCase.buf)), minio.PutObjectOptions{ServerSideEncryption: sse}) + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(testCase.buf), int64(len(testCase.buf)), minio.PutObjectOptions{ServerSideEncryption: sse}) if err != nil { logError(testName, function, args, startTime, "", "PutEncryptedObject failed", err) return } // Read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: sse}) + r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: sse}) if err != nil { logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) return @@ -3708,7 +3708,7 @@ func testSSECEncryptionFPut() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -3758,13 +3758,13 @@ func testSSECEncryptionFPut() { } file.Close() // Put encrypted data - if _, err = c.FPutObject(bucketName, objectName, fileName, minio.PutObjectOptions{ServerSideEncryption: sse}); err != nil { + if _, err = c.FPutObject(context.Background(), bucketName, objectName, fileName, minio.PutObjectOptions{ServerSideEncryption: sse}); err != nil { logError(testName, function, args, startTime, "", "FPutEncryptedObject failed", err) return } // Read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: sse}) + r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: sse}) if err != nil { logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) return @@ -3835,7 +3835,7 @@ func testSSES3EncryptionPutGet() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -3866,14 +3866,14 @@ func testSSES3EncryptionPutGet() { args["sse"] = sse // Put encrypted data - _, err = c.PutObject(bucketName, objectName, bytes.NewReader(testCase.buf), int64(len(testCase.buf)), minio.PutObjectOptions{ServerSideEncryption: sse}) + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(testCase.buf), int64(len(testCase.buf)), minio.PutObjectOptions{ServerSideEncryption: sse}) if err != nil { logError(testName, function, args, startTime, "", "PutEncryptedObject failed", err) return } // Read the data back without any encryption headers - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) return @@ -3947,7 +3947,7 @@ func testSSES3EncryptionFPut() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -3996,13 +3996,13 @@ func testSSES3EncryptionFPut() { } file.Close() // Put encrypted data - if _, err = c.FPutObject(bucketName, objectName, fileName, minio.PutObjectOptions{ServerSideEncryption: sse}); err != nil { + if _, err = c.FPutObject(context.Background(), bucketName, objectName, fileName, minio.PutObjectOptions{ServerSideEncryption: sse}); err != nil { logError(testName, function, args, startTime, "", "FPutEncryptedObject failed", err) return } // Read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) return @@ -4104,13 +4104,13 @@ func testBucketNotification() { bNotification.AddQueue(queueConfig) bNotification.RemoveQueueByArn(queueArn) - err = c.SetBucketNotification(bucketName, bNotification) + err = c.SetBucketNotification(context.Background(), bucketName, bNotification) if err != nil { logError(testName, function, args, startTime, "", "SetBucketNotification failed", err) return } - bNotification, err = c.GetBucketNotification(bucketName) + bNotification, err = c.GetBucketNotification(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketNotification failed", err) return @@ -4126,7 +4126,7 @@ func testBucketNotification() { return } - err = c.RemoveAllBucketNotification(bucketName) + err = c.RemoveAllBucketNotification(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "RemoveAllBucketNotification failed", err) return @@ -4177,7 +4177,7 @@ func testFunctional() { function = "MakeBucket(bucketName, region)" functionAll = "MakeBucket(bucketName, region)" args["bucketName"] = bucketName - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) @@ -4208,7 +4208,7 @@ func testFunctional() { args = map[string]interface{}{ "bucketName": bucketName, } - exists, err = c.BucketExists(bucketName) + exists, err = c.BucketExists(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "BucketExists failed", err) @@ -4219,30 +4219,13 @@ func testFunctional() { return } - // Verify if bucket exits and you have access with context. - function = "BucketExistsWithContext(ctx, bucketName)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - } - exists, err = c.BucketExistsWithContext(context.Background(), bucketName) - - if err != nil { - logError(testName, function, args, startTime, "", "BucketExistsWithContext failed", err) - return - } - if !exists { - logError(testName, function, args, startTime, "", "Could not find the bucket", err) - return - } - // Asserting the default bucket policy. - function = "GetBucketPolicy(bucketName)" + function = "GetBucketPolicy(ctx, bucketName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, } - nilPolicy, err := c.GetBucketPolicy(bucketName) + nilPolicy, err := c.GetBucketPolicy(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) return @@ -4262,18 +4245,18 @@ func testFunctional() { "bucketPolicy": readOnlyPolicy, } - err = c.SetBucketPolicy(bucketName, readOnlyPolicy) + err = c.SetBucketPolicy(context.Background(), bucketName, readOnlyPolicy) if err != nil { logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) return } // should return policy `readonly`. - function = "GetBucketPolicy(bucketName)" + function = "GetBucketPolicy(ctx, bucketName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, } - _, err = c.GetBucketPolicy(bucketName) + _, err = c.GetBucketPolicy(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) return @@ -4288,20 +4271,20 @@ func testFunctional() { "bucketName": bucketName, "bucketPolicy": writeOnlyPolicy, } - err = c.SetBucketPolicy(bucketName, writeOnlyPolicy) + err = c.SetBucketPolicy(context.Background(), bucketName, writeOnlyPolicy) if err != nil { logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) return } // should return policy `writeonly`. - function = "GetBucketPolicy(bucketName)" + function = "GetBucketPolicy(ctx, bucketName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, } - _, err = c.GetBucketPolicy(bucketName) + _, err = c.GetBucketPolicy(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) return @@ -4317,7 +4300,7 @@ func testFunctional() { "bucketName": bucketName, "bucketPolicy": readWritePolicy, } - err = c.SetBucketPolicy(bucketName, readWritePolicy) + err = c.SetBucketPolicy(context.Background(), bucketName, readWritePolicy) if err != nil { logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) @@ -4329,7 +4312,7 @@ func testFunctional() { args = map[string]interface{}{ "bucketName": bucketName, } - _, err = c.GetBucketPolicy(bucketName) + _, err = c.GetBucketPolicy(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) return @@ -4339,7 +4322,7 @@ func testFunctional() { function = "ListBuckets()" functionAll += ", " + function args = nil - buckets, err := c.ListBuckets() + buckets, err := c.ListBuckets(context.Background()) if len(buckets) == 0 { logError(testName, function, args, startTime, "", "Found bucket list to be empty", err) @@ -4350,21 +4333,6 @@ func testFunctional() { return } - // List all buckets with context. - function = "ListBucketsWithContext()" - functionAll += ", " + function - args = nil - buckets, err = c.ListBucketsWithContext(context.Background()) - - if len(buckets) == 0 { - logError(testName, function, args, startTime, "", "Found bucket list to be empty", err) - return - } - if err != nil { - logError(testName, function, args, startTime, "", "ListBucketsWithContext failed", err) - return - } - // Verify if previously created bucket is listed in list buckets. bucketFound := false for _, bucket := range buckets { @@ -4392,7 +4360,7 @@ func testFunctional() { "contentType": "", } - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) + n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -4409,7 +4377,7 @@ func testFunctional() { "contentType": "binary/octet-stream", } - n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + n, err = c.PutObject(context.Background(), bucketName, objectName+"-nolength", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -4435,7 +4403,7 @@ func testFunctional() { "isRecursive": isRecursive, } - for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { + for obj := range c.ListObjects(context.Background(), bucketName, objectName, isRecursive, doneCh) { if obj.Key == objectName { objFound = true break @@ -4456,7 +4424,7 @@ func testFunctional() { "isRecursive": isRecursive, } - for obj := range c.ListObjectsV2(bucketName, objectName, isRecursive, doneCh) { + for obj := range c.ListObjectsV2(context.Background(), bucketName, objectName, isRecursive, doneCh) { if obj.Key == objectName { objFound = true break @@ -4477,7 +4445,7 @@ func testFunctional() { "isRecursive": isRecursive, } - for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { + for objIncompl := range c.ListIncompleteUploads(context.Background(), bucketName, objectName, isRecursive, doneCh) { if objIncompl.Key != "" { incompObjNotFound = false break @@ -4494,7 +4462,7 @@ func testFunctional() { "bucketName": bucketName, "objectName": objectName, } - newReader, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + newReader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) @@ -4520,7 +4488,7 @@ func testFunctional() { "objectName": objectName, "fileName": fileName + "-f", } - err = c.FGetObject(bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) + err = c.FGetObject(context.Background(), bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FGetObject failed", err) @@ -4534,7 +4502,7 @@ func testFunctional() { "objectName": "", "expires": 3600 * time.Second, } - if _, err = c.PresignedHeadObject(bucketName, "", 3600*time.Second, nil); err == nil { + if _, err = c.PresignedHeadObject(context.Background(), bucketName, "", 3600*time.Second, nil); err == nil { logError(testName, function, args, startTime, "", "PresignedHeadObject success", err) return } @@ -4547,7 +4515,7 @@ func testFunctional() { "objectName": objectName, "expires": 3600 * time.Second, } - presignedHeadURL, err := c.PresignedHeadObject(bucketName, objectName, 3600*time.Second, nil) + presignedHeadURL, err := c.PresignedHeadObject(context.Background(), bucketName, objectName, 3600*time.Second, nil) if err != nil { logError(testName, function, args, startTime, "", "PresignedHeadObject failed", err) @@ -4597,7 +4565,7 @@ func testFunctional() { "objectName": "", "expires": 3600 * time.Second, } - _, err = c.PresignedGetObject(bucketName, "", 3600*time.Second, nil) + _, err = c.PresignedGetObject(context.Background(), bucketName, "", 3600*time.Second, nil) if err == nil { logError(testName, function, args, startTime, "", "PresignedGetObject success", err) return @@ -4611,7 +4579,7 @@ func testFunctional() { "objectName": objectName, "expires": 3600 * time.Second, } - presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second, nil) + presignedGetURL, err := c.PresignedGetObject(context.Background(), bucketName, objectName, 3600*time.Second, nil) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) @@ -4654,7 +4622,7 @@ func testFunctional() { "expires": 3600 * time.Second, "reqParams": reqParams, } - presignedGetURL, err = c.PresignedGetObject(bucketName, objectName, 3600*time.Second, reqParams) + presignedGetURL, err = c.PresignedGetObject(context.Background(), bucketName, objectName, 3600*time.Second, reqParams) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) @@ -4698,7 +4666,7 @@ func testFunctional() { "objectName": "", "expires": 3600 * time.Second, } - _, err = c.PresignedPutObject(bucketName, "", 3600*time.Second) + _, err = c.PresignedPutObject(context.Background(), bucketName, "", 3600*time.Second) if err == nil { logError(testName, function, args, startTime, "", "PresignedPutObject success", err) return @@ -4711,7 +4679,7 @@ func testFunctional() { "objectName": objectName + "-presigned", "expires": 3600 * time.Second, } - presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second) + presignedPutURL, err := c.PresignedPutObject(context.Background(), bucketName, objectName+"-presigned", 3600*time.Second) if err != nil { logError(testName, function, args, startTime, "", "PresignedPutObject failed", err) @@ -4732,7 +4700,7 @@ func testFunctional() { return } - newReader, err = c.GetObject(bucketName, objectName+"-presigned", minio.GetObjectOptions{}) + newReader, err = c.GetObject(context.Background(), bucketName, objectName+"-presigned", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject after PresignedPutObject failed", err) return @@ -4755,14 +4723,14 @@ func testFunctional() { "bucketName": bucketName, "objectName": objectName, } - err = c.RemoveObject(bucketName, objectName) + err = c.RemoveObject(context.Background(), bucketName, objectName) if err != nil { logError(testName, function, args, startTime, "", "RemoveObject failed", err) return } args["objectName"] = objectName + "-f" - err = c.RemoveObject(bucketName, objectName+"-f") + err = c.RemoveObject(context.Background(), bucketName, objectName+"-f") if err != nil { logError(testName, function, args, startTime, "", "RemoveObject failed", err) @@ -4770,7 +4738,7 @@ func testFunctional() { } args["objectName"] = objectName + "-nolength" - err = c.RemoveObject(bucketName, objectName+"-nolength") + err = c.RemoveObject(context.Background(), bucketName, objectName+"-nolength") if err != nil { logError(testName, function, args, startTime, "", "RemoveObject failed", err) @@ -4778,7 +4746,7 @@ func testFunctional() { } args["objectName"] = objectName + "-presigned" - err = c.RemoveObject(bucketName, objectName+"-presigned") + err = c.RemoveObject(context.Background(), bucketName, objectName+"-presigned") if err != nil { logError(testName, function, args, startTime, "", "RemoveObject failed", err) @@ -4790,13 +4758,13 @@ func testFunctional() { args = map[string]interface{}{ "bucketName": bucketName, } - err = c.RemoveBucket(bucketName) + err = c.RemoveBucket(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "RemoveBucket failed", err) return } - err = c.RemoveBucket(bucketName) + err = c.RemoveBucket(context.Background(), bucketName) if err == nil { logError(testName, function, args, startTime, "", "RemoveBucket did not fail for invalid bucket name", err) return @@ -4843,26 +4811,26 @@ func testGetObjectModified() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } - defer c.RemoveBucket(bucketName) + defer c.RemoveBucket(context.Background(), bucketName) // Upload an object. objectName := "myobject" args["objectName"] = objectName content := "helloworld" - _, err = c.PutObject(bucketName, objectName, strings.NewReader(content), int64(len(content)), minio.PutObjectOptions{ContentType: "application/text"}) + _, err = c.PutObject(context.Background(), bucketName, objectName, strings.NewReader(content), int64(len(content)), minio.PutObjectOptions{ContentType: "application/text"}) if err != nil { logError(testName, function, args, startTime, "", "Failed to upload "+objectName+", to bucket "+bucketName, err) return } - defer c.RemoveObject(bucketName, objectName) + defer c.RemoveObject(context.Background(), bucketName, objectName) - reader, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + reader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "Failed to GetObject "+objectName+", from bucket "+bucketName, err) return @@ -4879,7 +4847,7 @@ func testGetObjectModified() { // Upload different contents to the same object while object is being read. newContent := "goodbyeworld" - _, err = c.PutObject(bucketName, objectName, strings.NewReader(newContent), int64(len(newContent)), minio.PutObjectOptions{ContentType: "application/text"}) + _, err = c.PutObject(context.Background(), bucketName, objectName, strings.NewReader(newContent), int64(len(newContent)), minio.PutObjectOptions{ContentType: "application/text"}) if err != nil { logError(testName, function, args, startTime, "", "Failed to upload "+objectName+", to bucket "+bucketName, err) return @@ -4943,12 +4911,12 @@ func testPutObjectUploadSeekedObject() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } - defer c.RemoveBucket(bucketName) + defer c.RemoveBucket(context.Background(), bucketName) var tempfile *os.File @@ -4988,7 +4956,7 @@ func testPutObjectUploadSeekedObject() { return } - n, err := c.PutObject(bucketName, objectName, tempfile, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + n, err := c.PutObject(context.Background(), bucketName, objectName, tempfile, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -4999,7 +4967,7 @@ func testPutObjectUploadSeekedObject() { } tempfile.Close() - obj, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + obj, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return @@ -5016,7 +4984,7 @@ func testPutObjectUploadSeekedObject() { return } - n, err = c.PutObject(bucketName, objectName+"getobject", obj, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + n, err = c.PutObject(context.Background(), bucketName, objectName+"getobject", obj, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -5079,11 +5047,11 @@ func testMakeBucketErrorV2() { args["region"] = region // Make a new bucket in 'eu-west-1'. - if err = c.MakeBucket(bucketName, region); err != nil { + if err = c.MakeBucket(context.Background(), bucketName, region); err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } - if err = c.MakeBucket(bucketName, region); err == nil { + if err = c.MakeBucket(context.Background(), bucketName, region); err == nil { logError(testName, function, args, startTime, "", "MakeBucket did not fail for existing bucket name", err) return } @@ -5138,7 +5106,7 @@ func testGetObjectClosedTwiceV2() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -5153,7 +5121,7 @@ func testGetObjectClosedTwiceV2() { objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName - n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + n, err := c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -5165,7 +5133,7 @@ func testGetObjectClosedTwiceV2() { } // Read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return @@ -5238,7 +5206,7 @@ func testFPutObjectV2() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -5275,7 +5243,7 @@ func testFPutObjectV2() { args["fileName"] = file.Name() // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) - n, err = c.FPutObject(bucketName, objectName+"-standard", file.Name(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) + n, err = c.FPutObject(context.Background(), bucketName, objectName+"-standard", file.Name(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject failed", err) return @@ -5289,7 +5257,7 @@ func testFPutObjectV2() { args["objectName"] = objectName + "-Octet" args["contentType"] = "" - n, err = c.FPutObject(bucketName, objectName+"-Octet", file.Name(), minio.PutObjectOptions{}) + n, err = c.FPutObject(context.Background(), bucketName, objectName+"-Octet", file.Name(), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject failed", err) return @@ -5312,7 +5280,7 @@ func testFPutObjectV2() { args["contentType"] = "" args["fileName"] = fileName + ".gtar" - n, err = c.FPutObject(bucketName, objectName+"-GTar", fileName+".gtar", minio.PutObjectOptions{}) + n, err = c.FPutObject(context.Background(), bucketName, objectName+"-GTar", fileName+".gtar", minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject failed", err) return @@ -5323,7 +5291,7 @@ func testFPutObjectV2() { } // Check headers - rStandard, err := c.StatObject(bucketName, objectName+"-standard", minio.StatObjectOptions{}) + rStandard, err := c.StatObject(context.Background(), bucketName, objectName+"-standard", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return @@ -5333,7 +5301,7 @@ func testFPutObjectV2() { return } - rOctet, err := c.StatObject(bucketName, objectName+"-Octet", minio.StatObjectOptions{}) + rOctet, err := c.StatObject(context.Background(), bucketName, objectName+"-Octet", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return @@ -5343,17 +5311,7 @@ func testFPutObjectV2() { return } - rGTar, err := c.StatObject(bucketName, objectName+"-GTar", minio.StatObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "StatObject failed", err) - return - } - if rGTar.ContentType != "application/x-gtar" && rGTar.ContentType != "application/octet-stream" { - logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/x-gtar , got "+rGTar.ContentType, err) - return - } - - rGTar, err = c.StatObjectWithContext(context.Background(), bucketName, objectName+"-GTar", minio.StatObjectOptions{}) + rGTar, err := c.StatObject(context.Background(), bucketName, objectName+"-GTar", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return @@ -5415,7 +5373,7 @@ func testMakeBucketRegionsV2() { args["bucketName"] = bucketName // Make a new bucket in 'eu-central-1'. - if err = c.MakeBucket(bucketName, "eu-west-1"); err != nil { + if err = c.MakeBucket(context.Background(), bucketName, "eu-west-1"); err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } @@ -5428,7 +5386,7 @@ func testMakeBucketRegionsV2() { // Make a new bucket with '.' in its name, in 'us-west-2'. This // request is internally staged into a path style instead of // virtual host style. - if err = c.MakeBucket(bucketName+".withperiod", "us-west-2"); err != nil { + if err = c.MakeBucket(context.Background(), bucketName+".withperiod", "us-west-2"); err != nil { args["bucketName"] = bucketName + ".withperiod" args["region"] = "us-west-2" logError(testName, function, args, startTime, "", "MakeBucket failed", err) @@ -5478,7 +5436,7 @@ func testGetObjectReadSeekFunctionalV2() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -5499,7 +5457,7 @@ func testGetObjectReadSeekFunctionalV2() { } // Save the data. - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -5511,7 +5469,7 @@ func testGetObjectReadSeekFunctionalV2() { } // Read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return @@ -5642,7 +5600,7 @@ func testGetObjectReadAtFunctionalV2() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -5663,7 +5621,7 @@ func testGetObjectReadAtFunctionalV2() { } // Save the data - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -5675,7 +5633,7 @@ func testGetObjectReadAtFunctionalV2() { } // Read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return @@ -5811,14 +5769,14 @@ func testCopyObjectV2() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } // Make a new bucket in 'us-east-1' (destination bucket). - err = c.MakeBucket(bucketName+"-copy", "us-east-1") + err = c.MakeBucket(context.Background(), bucketName+"-copy", "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -5831,7 +5789,7 @@ func testCopyObjectV2() { // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + n, err := c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -5842,7 +5800,7 @@ func testCopyObjectV2() { return } - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return @@ -5902,20 +5860,20 @@ func testCopyObjectV2() { } // Perform the Copy - err = c.CopyObject(dst, src) + err = c.CopyObject(context.Background(), dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } // Source object - r, err = c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + r, err = c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } // Destination object - readerCopy, err := c.GetObject(bucketName+"-copy", objectName+"-copy", minio.GetObjectOptions{}) + readerCopy, err := c.GetObject(context.Background(), bucketName+"-copy", objectName+"-copy", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return @@ -5954,7 +5912,7 @@ func testCopyObjectV2() { } // Perform the Copy which should fail - err = c.CopyObject(dst, src) + err = c.CopyObject(context.Background(), dst, src) if err == nil { logError(testName, function, args, startTime, "", "CopyObject did not fail for invalid conditions", err) return @@ -5983,7 +5941,7 @@ func testComposeObjectErrorCasesWrapper(c *minio.Client) { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). - err := c.MakeBucket(bucketName, "us-east-1") + err := c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) @@ -6004,7 +5962,7 @@ func testComposeObjectErrorCasesWrapper(c *minio.Client) { // Just explain about srcArr in args["sourceList"] // to stop having 10,001 null headers logged args["sourceList"] = "source array of 10,001 elements" - if err := c.ComposeObject(dst, srcSlice); err == nil { + if err := c.ComposeObject(context.Background(), dst, srcSlice); err == nil { logError(testName, function, args, startTime, "", "Expected error in ComposeObject", err) return } else if err.Error() != "There must be as least one and up to 10000 source objects." { @@ -6017,7 +5975,7 @@ func testComposeObjectErrorCasesWrapper(c *minio.Client) { // 1. Create the source object. const badSrcSize = 5 * 1024 * 1024 buf := bytes.Repeat([]byte("1"), badSrcSize) - _, err = c.PutObject(bucketName, "badObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) + _, err = c.PutObject(context.Background(), bucketName, "badObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -6031,7 +5989,7 @@ func testComposeObjectErrorCasesWrapper(c *minio.Client) { return } // 3. ComposeObject call should fail. - if err := c.ComposeObject(dst, []minio.SourceInfo{badSrc}); err == nil { + if err := c.ComposeObject(context.Background(), dst, []minio.SourceInfo{badSrc}); err == nil { logError(testName, function, args, startTime, "", "ComposeObject expected to fail", err) return } else if !strings.Contains(err.Error(), "has invalid segment-to-copy") { @@ -6084,7 +6042,7 @@ func testComposeMultipleSources(c *minio.Client) { // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). - err := c.MakeBucket(bucketName, "us-east-1") + err := c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -6093,7 +6051,7 @@ func testComposeMultipleSources(c *minio.Client) { // Upload a small source object const srcSize = 1024 * 1024 * 5 buf := bytes.Repeat([]byte("1"), srcSize) - _, err = c.PutObject(bucketName, "srcObject", bytes.NewReader(buf), int64(srcSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + _, err = c.PutObject(context.Background(), bucketName, "srcObject", bytes.NewReader(buf), int64(srcSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -6119,13 +6077,13 @@ func testComposeMultipleSources(c *minio.Client) { logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) return } - err = c.ComposeObject(dst, srcs) + err = c.ComposeObject(context.Background(), dst, srcs) if err != nil { logError(testName, function, args, startTime, "", "ComposeObject failed", err) return } - objProps, err := c.StatObject(bucketName, "dstObject", minio.StatObjectOptions{}) + objProps, err := c.StatObject(context.Background(), bucketName, "dstObject", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return @@ -6189,7 +6147,7 @@ func testEncryptedEmptyObject() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -6201,7 +6159,7 @@ func testEncryptedEmptyObject() { const srcSize = 0 var buf []byte // Empty buffer args["objectName"] = "object" - _, err = c.PutObject(bucketName, "object", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ServerSideEncryption: sse}) + _, err = c.PutObject(context.Background(), bucketName, "object", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ServerSideEncryption: sse}) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return @@ -6216,7 +6174,7 @@ func testEncryptedEmptyObject() { return } srcInfo := minio.NewSourceInfo(bucketName, "object", sse) - if err = c.CopyObject(dstInfo, srcInfo); err != nil { + if err = c.CopyObject(context.Background(), dstInfo, srcInfo); err != nil { function = "CopyObject(dstInfo, srcInfo)" logError(testName, function, map[string]interface{}{}, startTime, "", "CopyObject failed", err) return @@ -6233,14 +6191,14 @@ func testEncryptedEmptyObject() { } srcInfo = minio.NewSourceInfo(bucketName, "new-object", sse) - if err = c.CopyObject(dstInfo, srcInfo); err != nil { + if err = c.CopyObject(context.Background(), dstInfo, srcInfo); err != nil { function = "CopyObject(dstInfo, srcInfo)" logError(testName, function, map[string]interface{}{}, startTime, "", "CopyObject with key rotation failed", err) return } // 4. Download the object. - reader, err := c.GetObject(bucketName, "new-object", minio.GetObjectOptions{ServerSideEncryption: newSSE}) + reader, err := c.GetObject(context.Background(), bucketName, "new-object", minio.GetObjectOptions{ServerSideEncryption: newSSE}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return @@ -6275,7 +6233,7 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, var srcEncryption, dstEncryption encrypt.ServerSide // Make a new bucket in 'us-east-1' (source bucket). - err := c.MakeBucket(bucketName, "us-east-1") + err := c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -6284,7 +6242,7 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, // 1. create an sse-c encrypted object to copy by uploading const srcSize = 1024 * 1024 buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB - _, err = c.PutObject(bucketName, "srcObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ + _, err = c.PutObject(context.Background(), bucketName, "srcObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ ServerSideEncryption: sseSrc, }) if err != nil { @@ -6306,7 +6264,7 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, } args["destination"] = dst - err = c.CopyObject(dst, src) + err = c.CopyObject(context.Background(), dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return @@ -6317,7 +6275,7 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, } // 3. get copied object and check if content is equal coreClient := minio.Core{c} - reader, _, _, err := coreClient.GetObject(bucketName, "dstObject", minio.GetObjectOptions{ServerSideEncryption: dstEncryption}) + reader, _, _, err := coreClient.GetObject(context.Background(), bucketName, "dstObject", minio.GetObjectOptions{ServerSideEncryption: dstEncryption}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return @@ -6350,14 +6308,14 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, } args["destination"] = dst - err = c.CopyObject(dst, src) + err = c.CopyObject(context.Background(), dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } // Get copied object and check if content is equal - reader, _, _, err = coreClient.GetObject(bucketName, "srcObject", minio.GetObjectOptions{ServerSideEncryption: newSSE}) + reader, _, _, err = coreClient.GetObject(context.Background(), bucketName, "srcObject", minio.GetObjectOptions{ServerSideEncryption: newSSE}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return @@ -6383,7 +6341,7 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, src = minio.NewSourceInfo(bucketName, "srcObject", newSSE) args["source"] = src - err = c.CopyObject(dst, src) + err = c.CopyObject(context.Background(), dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject Key rotation failed", err) return @@ -6391,7 +6349,7 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, } // Get copied decrypted object and check if content is equal - reader, _, _, err = coreClient.GetObject(bucketName, "srcObject", minio.GetObjectOptions{}) + reader, _, _, err = coreClient.GetObject(context.Background(), bucketName, "srcObject", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return @@ -6716,13 +6674,13 @@ func testDecryptedCopyObject() { } bucketName, objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-"), "object" - if err = c.MakeBucket(bucketName, "us-east-1"); err != nil { + if err = c.MakeBucket(context.Background(), bucketName, "us-east-1"); err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } encryption := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)) - _, err = c.PutObject(bucketName, objectName, bytes.NewReader(bytes.Repeat([]byte("a"), 1024*1024)), 1024*1024, minio.PutObjectOptions{ + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(bytes.Repeat([]byte("a"), 1024*1024)), 1024*1024, minio.PutObjectOptions{ ServerSideEncryption: encryption, }) if err != nil { @@ -6739,11 +6697,11 @@ func testDecryptedCopyObject() { } args["destination"] = dst - if err = c.CopyObject(dst, src); err != nil { + if err = c.CopyObject(context.Background(), dst, src); err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } - if _, err = c.GetObject(bucketName, "decrypted-"+objectName, minio.GetObjectOptions{}); err != nil { + if _, err = c.GetObject(context.Background(), bucketName, "decrypted-"+objectName, minio.GetObjectOptions{}); err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } @@ -6782,7 +6740,7 @@ func testSSECMultipartEncryptedToSSECCopyObjectPart() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) } @@ -6796,33 +6754,33 @@ func testSSECMultipartEncryptedToSSECCopyObjectPart() { srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) // Upload a 6MB object using multipart mechanism - uploadID, err := c.NewMultipartUpload(bucketName, objectName, minio.PutObjectOptions{ServerSideEncryption: srcencryption}) + uploadID, err := c.NewMultipartUpload(context.Background(), bucketName, objectName, minio.PutObjectOptions{ServerSideEncryption: srcencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) } var completeParts []minio.CompletePart - part, err := c.PutObjectPart(bucketName, objectName, uploadID, 1, bytes.NewReader(buf[:5*1024*1024]), 5*1024*1024, "", "", srcencryption) + part, err := c.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, bytes.NewReader(buf[:5*1024*1024]), 5*1024*1024, "", "", srcencryption) if err != nil { logError(testName, function, args, startTime, "", "PutObjectPart call failed", err) } completeParts = append(completeParts, minio.CompletePart{PartNumber: part.PartNumber, ETag: part.ETag}) - part, err = c.PutObjectPart(bucketName, objectName, uploadID, 2, bytes.NewReader(buf[5*1024*1024:]), 1024*1024, "", "", srcencryption) + part, err = c.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 2, bytes.NewReader(buf[5*1024*1024:]), 1024*1024, "", "", srcencryption) if err != nil { logError(testName, function, args, startTime, "", "PutObjectPart call failed", err) } completeParts = append(completeParts, minio.CompletePart{PartNumber: part.PartNumber, ETag: part.ETag}) // Complete the multipart upload - _, err = c.CompleteMultipartUpload(bucketName, objectName, uploadID, completeParts) + _, err = c.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, completeParts) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) } // Stat the object and check its length matches - objInfo, err := c.StatObject(bucketName, objectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: srcencryption}}) + objInfo, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: srcencryption}}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -6831,7 +6789,7 @@ func testSSECMultipartEncryptedToSSECCopyObjectPart() { destObjectName := objectName + "-dest" dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName)) - uploadID, err = c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) + uploadID, err = c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) } @@ -6850,31 +6808,31 @@ func testSSECMultipartEncryptedToSSECCopyObjectPart() { metadata["x-amz-copy-source-if-match"] = objInfo.ETag // First of three parts - fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) + fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Second of three parts - sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) + sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Last of three parts - lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) + lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Complete the multipart upload - _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) } // Stat the object and check its length matches - objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) + objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -6886,7 +6844,7 @@ func testSSECMultipartEncryptedToSSECCopyObjectPart() { // Now we read the data back getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} getOpts.SetRange(0, 6*1024*1024-1) - r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -6900,7 +6858,7 @@ func testSSECMultipartEncryptedToSSECCopyObjectPart() { } getOpts.SetRange(6*1024*1024, 0) - r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -6954,7 +6912,7 @@ func testSSECEncryptedToSSECCopyObjectPart() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) } @@ -6973,7 +6931,7 @@ func testSSECEncryptedToSSECCopyObjectPart() { UserMetadata: putmetadata, ServerSideEncryption: srcencryption, } - objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) + objInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } @@ -6986,7 +6944,7 @@ func testSSECEncryptedToSSECCopyObjectPart() { destObjectName := objectName + "-dest" dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName)) - uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) + uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) } @@ -7005,31 +6963,31 @@ func testSSECEncryptedToSSECCopyObjectPart() { metadata["x-amz-copy-source-if-match"] = objInfo.ETag // First of three parts - fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) + fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Second of three parts - sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) + sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Last of three parts - lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) + lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Complete the multipart upload - _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) } // Stat the object and check its length matches - objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) + objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -7041,7 +6999,7 @@ func testSSECEncryptedToSSECCopyObjectPart() { // Now we read the data back getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} getOpts.SetRange(0, 5*1024*1024-1) - r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -7055,7 +7013,7 @@ func testSSECEncryptedToSSECCopyObjectPart() { } getOpts.SetRange(5*1024*1024, 0) - r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -7109,7 +7067,7 @@ func testSSECEncryptedToUnencryptedCopyPart() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) } @@ -7128,7 +7086,7 @@ func testSSECEncryptedToUnencryptedCopyPart() { }, ServerSideEncryption: srcencryption, } - objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) + objInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } @@ -7141,7 +7099,7 @@ func testSSECEncryptedToUnencryptedCopyPart() { destObjectName := objectName + "-dest" var dstencryption encrypt.ServerSide - uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) + uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) } @@ -7159,31 +7117,31 @@ func testSSECEncryptedToUnencryptedCopyPart() { metadata["x-amz-copy-source-if-match"] = objInfo.ETag // First of three parts - fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) + fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Second of three parts - sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) + sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Last of three parts - lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) + lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Complete the multipart upload - _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) } // Stat the object and check its length matches - objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) + objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -7195,7 +7153,7 @@ func testSSECEncryptedToUnencryptedCopyPart() { // Now we read the data back getOpts := minio.GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) - r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -7209,7 +7167,7 @@ func testSSECEncryptedToUnencryptedCopyPart() { } getOpts.SetRange(5*1024*1024, 0) - r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -7263,7 +7221,7 @@ func testSSECEncryptedToSSES3CopyObjectPart() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) } @@ -7283,7 +7241,7 @@ func testSSECEncryptedToSSES3CopyObjectPart() { ServerSideEncryption: srcencryption, } - objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) + objInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } @@ -7296,7 +7254,7 @@ func testSSECEncryptedToSSES3CopyObjectPart() { destObjectName := objectName + "-dest" dstencryption := encrypt.NewSSE() - uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) + uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) } @@ -7316,31 +7274,31 @@ func testSSECEncryptedToSSES3CopyObjectPart() { metadata["x-amz-copy-source-if-match"] = objInfo.ETag // First of three parts - fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) + fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Second of three parts - sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) + sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Last of three parts - lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) + lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Complete the multipart upload - _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) } // Stat the object and check its length matches - objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) + objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -7352,7 +7310,7 @@ func testSSECEncryptedToSSES3CopyObjectPart() { // Now we read the data back getOpts := minio.GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) - r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -7366,7 +7324,7 @@ func testSSECEncryptedToSSES3CopyObjectPart() { } getOpts.SetRange(5*1024*1024, 0) - r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -7420,7 +7378,7 @@ func testUnencryptedToSSECCopyObjectPart() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) } @@ -7437,7 +7395,7 @@ func testUnencryptedToSSECCopyObjectPart() { opts := minio.PutObjectOptions{ UserMetadata: putmetadata, } - objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) + objInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } @@ -7450,7 +7408,7 @@ func testUnencryptedToSSECCopyObjectPart() { destObjectName := objectName + "-dest" dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName)) - uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) + uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) } @@ -7468,31 +7426,31 @@ func testUnencryptedToSSECCopyObjectPart() { metadata["x-amz-copy-source-if-match"] = objInfo.ETag // First of three parts - fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) + fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Second of three parts - sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) + sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Last of three parts - lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) + lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Complete the multipart upload - _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) } // Stat the object and check its length matches - objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) + objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -7504,7 +7462,7 @@ func testUnencryptedToSSECCopyObjectPart() { // Now we read the data back getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} getOpts.SetRange(0, 5*1024*1024-1) - r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -7518,7 +7476,7 @@ func testUnencryptedToSSECCopyObjectPart() { } getOpts.SetRange(5*1024*1024, 0) - r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -7572,7 +7530,7 @@ func testUnencryptedToUnencryptedCopyPart() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) } @@ -7588,7 +7546,7 @@ func testUnencryptedToUnencryptedCopyPart() { opts := minio.PutObjectOptions{ UserMetadata: putmetadata, } - objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) + objInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } @@ -7600,7 +7558,7 @@ func testUnencryptedToUnencryptedCopyPart() { destBucketName := bucketName destObjectName := objectName + "-dest" - uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{}) + uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) } @@ -7617,31 +7575,31 @@ func testUnencryptedToUnencryptedCopyPart() { metadata["x-amz-copy-source-if-match"] = objInfo.ETag // First of three parts - fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) + fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Second of three parts - sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) + sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Last of three parts - lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) + lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Complete the multipart upload - _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) } // Stat the object and check its length matches - objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) + objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -7653,7 +7611,7 @@ func testUnencryptedToUnencryptedCopyPart() { // Now we read the data back getOpts := minio.GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) - r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -7667,7 +7625,7 @@ func testUnencryptedToUnencryptedCopyPart() { } getOpts.SetRange(5*1024*1024, 0) - r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -7721,7 +7679,7 @@ func testUnencryptedToSSES3CopyObjectPart() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) } @@ -7736,7 +7694,7 @@ func testUnencryptedToSSES3CopyObjectPart() { "Content-Type": "binary/octet-stream", }, } - objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) + objInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } @@ -7749,7 +7707,7 @@ func testUnencryptedToSSES3CopyObjectPart() { destObjectName := objectName + "-dest" dstencryption := encrypt.NewSSE() - uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) + uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) } @@ -7768,31 +7726,31 @@ func testUnencryptedToSSES3CopyObjectPart() { metadata["x-amz-copy-source-if-match"] = objInfo.ETag // First of three parts - fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) + fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Second of three parts - sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) + sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Last of three parts - lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) + lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Complete the multipart upload - _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) } // Stat the object and check its length matches - objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) + objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -7804,7 +7762,7 @@ func testUnencryptedToSSES3CopyObjectPart() { // Now we read the data back getOpts := minio.GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) - r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -7818,7 +7776,7 @@ func testUnencryptedToSSES3CopyObjectPart() { } getOpts.SetRange(5*1024*1024, 0) - r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -7872,7 +7830,7 @@ func testSSES3EncryptedToSSECCopyObjectPart() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) } @@ -7890,7 +7848,7 @@ func testSSES3EncryptedToSSECCopyObjectPart() { }, ServerSideEncryption: srcEncryption, } - objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) + objInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } @@ -7903,7 +7861,7 @@ func testSSES3EncryptedToSSECCopyObjectPart() { destObjectName := objectName + "-dest" dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName)) - uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) + uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) } @@ -7921,31 +7879,31 @@ func testSSES3EncryptedToSSECCopyObjectPart() { metadata["x-amz-copy-source-if-match"] = objInfo.ETag // First of three parts - fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) + fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Second of three parts - sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) + sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Last of three parts - lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) + lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Complete the multipart upload - _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) } // Stat the object and check its length matches - objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) + objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -7957,7 +7915,7 @@ func testSSES3EncryptedToSSECCopyObjectPart() { // Now we read the data back getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} getOpts.SetRange(0, 5*1024*1024-1) - r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -7971,7 +7929,7 @@ func testSSES3EncryptedToSSECCopyObjectPart() { } getOpts.SetRange(5*1024*1024, 0) - r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -8025,7 +7983,7 @@ func testSSES3EncryptedToUnencryptedCopyPart() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) } @@ -8042,7 +8000,7 @@ func testSSES3EncryptedToUnencryptedCopyPart() { }, ServerSideEncryption: srcEncryption, } - objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) + objInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } @@ -8054,7 +8012,7 @@ func testSSES3EncryptedToUnencryptedCopyPart() { destBucketName := bucketName destObjectName := objectName + "-dest" - uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{}) + uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) } @@ -8071,31 +8029,31 @@ func testSSES3EncryptedToUnencryptedCopyPart() { metadata["x-amz-copy-source-if-match"] = objInfo.ETag // First of three parts - fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) + fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Second of three parts - sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) + sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Last of three parts - lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) + lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Complete the multipart upload - _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) } // Stat the object and check its length matches - objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) + objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -8107,7 +8065,7 @@ func testSSES3EncryptedToUnencryptedCopyPart() { // Now we read the data back getOpts := minio.GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) - r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -8121,7 +8079,7 @@ func testSSES3EncryptedToUnencryptedCopyPart() { } getOpts.SetRange(5*1024*1024, 0) - r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -8175,7 +8133,7 @@ func testSSES3EncryptedToSSES3CopyObjectPart() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) } @@ -8193,7 +8151,7 @@ func testSSES3EncryptedToSSES3CopyObjectPart() { ServerSideEncryption: srcEncryption, } - objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) + objInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } @@ -8206,7 +8164,7 @@ func testSSES3EncryptedToSSES3CopyObjectPart() { destObjectName := objectName + "-dest" dstencryption := encrypt.NewSSE() - uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) + uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) } @@ -8225,31 +8183,31 @@ func testSSES3EncryptedToSSES3CopyObjectPart() { metadata["x-amz-copy-source-if-match"] = objInfo.ETag // First of three parts - fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) + fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Second of three parts - sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) + sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Last of three parts - lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) + lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } // Complete the multipart upload - _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) } // Stat the object and check its length matches - objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) + objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -8261,7 +8219,7 @@ func testSSES3EncryptedToSSES3CopyObjectPart() { // Now we read the data back getOpts := minio.GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) - r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -8275,7 +8233,7 @@ func testSSES3EncryptedToSSES3CopyObjectPart() { } getOpts.SetRange(5*1024*1024, 0) - r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -8328,14 +8286,14 @@ func testUserMetadataCopyingWrapper(c *minio.Client) { // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). - err := c.MakeBucket(bucketName, "us-east-1") + err := c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } fetchMeta := func(object string) (h http.Header) { - objInfo, err := c.StatObject(bucketName, object, minio.StatObjectOptions{}) + objInfo, err := c.StatObject(context.Background(), bucketName, object, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return @@ -8358,7 +8316,7 @@ func testUserMetadataCopyingWrapper(c *minio.Client) { metadata.Set("x-amz-meta-myheader", "myvalue") m := make(map[string]string) m["x-amz-meta-myheader"] = "myvalue" - _, err = c.PutObject(bucketName, "srcObject", + _, err = c.PutObject(context.Background(), bucketName, "srcObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{UserMetadata: m}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectWithMetadata failed", err) @@ -8382,7 +8340,7 @@ func testUserMetadataCopyingWrapper(c *minio.Client) { // the headers on the copy. args["source"] = src args["destination"] = dst1 - err = c.CopyObject(dst1, src) + err = c.CopyObject(context.Background(), dst1, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return @@ -8407,7 +8365,7 @@ func testUserMetadataCopyingWrapper(c *minio.Client) { // copies metadata. args["source"] = src args["destination"] = dst2 - err = c.CopyObject(dst2, src) + err = c.CopyObject(context.Background(), dst2, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return @@ -8433,7 +8391,7 @@ func testUserMetadataCopyingWrapper(c *minio.Client) { function = "ComposeObject(destination, sources)" args["source"] = srcs args["destination"] = dst3 - err = c.ComposeObject(dst3, srcs) + err = c.ComposeObject(context.Background(), dst3, srcs) if err != nil { logError(testName, function, args, startTime, "", "ComposeObject failed", err) return @@ -8459,7 +8417,7 @@ func testUserMetadataCopyingWrapper(c *minio.Client) { function = "ComposeObject(destination, sources)" args["source"] = srcs args["destination"] = dst4 - err = c.ComposeObject(dst4, srcs) + err = c.ComposeObject(context.Background(), dst4, srcs) if err != nil { logError(testName, function, args, startTime, "", "ComposeObject failed", err) return @@ -8527,14 +8485,14 @@ func testStorageClassMetadataPutObject() { // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } fetchMeta := func(object string) (h http.Header) { - objInfo, err := c.StatObject(bucketName, object, minio.StatObjectOptions{}) + objInfo, err := c.StatObject(context.Background(), bucketName, object, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return @@ -8558,7 +8516,7 @@ func testStorageClassMetadataPutObject() { const srcSize = 1024 * 1024 buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 1MiB - _, err = c.PutObject(bucketName, "srcObjectRRSClass", + _, err = c.PutObject(context.Background(), bucketName, "srcObjectRRSClass", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "REDUCED_REDUNDANCY"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) @@ -8577,7 +8535,7 @@ func testStorageClassMetadataPutObject() { metadata = make(http.Header) metadata.Set("x-amz-storage-class", "STANDARD") - _, err = c.PutObject(bucketName, "srcObjectSSClass", + _, err = c.PutObject(context.Background(), bucketName, "srcObjectSSClass", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "STANDARD"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) @@ -8617,7 +8575,7 @@ func testStorageClassInvalidMetadataPutObject() { // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -8626,7 +8584,7 @@ func testStorageClassInvalidMetadataPutObject() { const srcSize = 1024 * 1024 buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 1MiB - _, err = c.PutObject(bucketName, "srcObjectRRSClass", + _, err = c.PutObject(context.Background(), bucketName, "srcObjectRRSClass", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "INVALID_STORAGE_CLASS"}) if err == nil { logError(testName, function, args, startTime, "", "PutObject with invalid storage class passed, was expected to fail", err) @@ -8662,14 +8620,14 @@ func testStorageClassMetadataCopyObject() { // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } fetchMeta := func(object string) (h http.Header) { - objInfo, err := c.StatObject(bucketName, object, minio.StatObjectOptions{}) + objInfo, err := c.StatObject(context.Background(), bucketName, object, minio.StatObjectOptions{}) args["bucket"] = bucketName args["object"] = object if err != nil { @@ -8696,7 +8654,7 @@ func testStorageClassMetadataCopyObject() { buf := bytes.Repeat([]byte("abcde"), srcSize) // Put an object with RRS Storage class - _, err = c.PutObject(bucketName, "srcObjectRRSClass", + _, err = c.PutObject(context.Background(), bucketName, "srcObjectRRSClass", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "REDUCED_REDUNDANCY"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) @@ -8706,7 +8664,7 @@ func testStorageClassMetadataCopyObject() { // Make server side copy of object uploaded in previous step src := minio.NewSourceInfo(bucketName, "srcObjectRRSClass", nil) dst, err := minio.NewDestinationInfo(bucketName, "srcObjectRRSClassCopy", nil, nil) - if err = c.CopyObject(dst, src); err != nil { + if err = c.CopyObject(context.Background(), dst, src); err != nil { logError(testName, function, args, startTime, "", "CopyObject failed on RRS", err) } @@ -8723,7 +8681,7 @@ func testStorageClassMetadataCopyObject() { metadata.Set("x-amz-storage-class", "STANDARD") // Put an object with Standard Storage class - _, err = c.PutObject(bucketName, "srcObjectSSClass", + _, err = c.PutObject(context.Background(), bucketName, "srcObjectSSClass", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "STANDARD"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) @@ -8733,7 +8691,7 @@ func testStorageClassMetadataCopyObject() { // Make server side copy of object uploaded in previous step src = minio.NewSourceInfo(bucketName, "srcObjectSSClass", nil) dst, err = minio.NewDestinationInfo(bucketName, "srcObjectSSClassCopy", nil, nil) - if err = c.CopyObject(dst, src); err != nil { + if err = c.CopyObject(context.Background(), dst, src); err != nil { logError(testName, function, args, startTime, "", "CopyObject failed on SS", err) } // Fetch the meta data of copied object @@ -8788,7 +8746,7 @@ func testPutObjectNoLengthV2() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -8803,7 +8761,7 @@ func testPutObjectNoLengthV2() { args["size"] = bufSize // Upload an object. - n, err := c.PutObject(bucketName, objectName, reader, -1, minio.PutObjectOptions{}) + n, err := c.PutObject(context.Background(), bucketName, objectName, reader, -1, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectWithSize failed", err) @@ -8862,7 +8820,7 @@ func testPutObjectsUnknownV2() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -8885,7 +8843,7 @@ func testPutObjectsUnknownV2() { objectName := fmt.Sprintf("%sunique%d", bucketName, i) args["objectName"] = objectName - n, err := c.PutObject(bucketName, objectName, rpipe, -1, minio.PutObjectOptions{}) + n, err := c.PutObject(context.Background(), bucketName, objectName, rpipe, -1, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectStreaming failed", err) return @@ -8946,7 +8904,7 @@ func testPutObject0ByteV2() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -8957,7 +8915,7 @@ func testPutObject0ByteV2() { args["opts"] = minio.PutObjectOptions{} // Upload an object. - n, err := c.PutObject(bucketName, objectName, bytes.NewReader([]byte("")), 0, minio.PutObjectOptions{}) + n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader([]byte("")), 0, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectWithSize failed", err) @@ -9062,7 +9020,7 @@ func testFunctionalV2() { "bucketName": bucketName, "location": location, } - err = c.MakeBucket(bucketName, location) + err = c.MakeBucket(context.Background(), bucketName, location) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -9092,7 +9050,7 @@ func testFunctionalV2() { args = map[string]interface{}{ "bucketName": bucketName, } - exists, err = c.BucketExists(bucketName) + exists, err = c.BucketExists(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "BucketExists failed", err) return @@ -9112,7 +9070,7 @@ func testFunctionalV2() { "bucketName": bucketName, "bucketPolicy": readWritePolicy, } - err = c.SetBucketPolicy(bucketName, readWritePolicy) + err = c.SetBucketPolicy(context.Background(), bucketName, readWritePolicy) if err != nil { logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) @@ -9123,7 +9081,7 @@ func testFunctionalV2() { function = "ListBuckets()" functionAll += ", " + function args = nil - buckets, err := c.ListBuckets() + buckets, err := c.ListBuckets(context.Background()) if len(buckets) == 0 { logError(testName, function, args, startTime, "", "List buckets cannot be empty", err) return @@ -9157,7 +9115,7 @@ func testFunctionalV2() { "objectName": objectName, "contentType": "", } - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) + n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -9169,7 +9127,7 @@ func testFunctionalV2() { objectNameNoLength := objectName + "-nolength" args["objectName"] = objectNameNoLength - n, err = c.PutObject(bucketName, objectNameNoLength, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + n, err = c.PutObject(context.Background(), bucketName, objectNameNoLength, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -9193,7 +9151,7 @@ func testFunctionalV2() { "objectName": objectName, "isRecursive": isRecursive, } - for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { + for obj := range c.ListObjects(context.Background(), bucketName, objectName, isRecursive, doneCh) { if obj.Key == objectName { objFound = true break @@ -9212,7 +9170,7 @@ func testFunctionalV2() { "objectName": objectName, "isRecursive": isRecursive, } - for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { + for objIncompl := range c.ListIncompleteUploads(context.Background(), bucketName, objectName, isRecursive, doneCh) { if objIncompl.Key != "" { incompObjNotFound = false break @@ -9229,7 +9187,7 @@ func testFunctionalV2() { "bucketName": bucketName, "objectName": objectName, } - newReader, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + newReader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return @@ -9254,7 +9212,7 @@ func testFunctionalV2() { "objectName": objectName, "fileName": fileName + "-f", } - err = c.FGetObject(bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) + err = c.FGetObject(context.Background(), bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FgetObject failed", err) return @@ -9268,7 +9226,7 @@ func testFunctionalV2() { "objectName": objectName, "expires": 3600 * time.Second, } - presignedHeadURL, err := c.PresignedHeadObject(bucketName, objectName, 3600*time.Second, nil) + presignedHeadURL, err := c.PresignedHeadObject(context.Background(), bucketName, objectName, 3600*time.Second, nil) if err != nil { logError(testName, function, args, startTime, "", "PresignedHeadObject failed", err) return @@ -9318,7 +9276,7 @@ func testFunctionalV2() { "objectName": objectName, "expires": 3600 * time.Second, } - presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second, nil) + presignedGetURL, err := c.PresignedGetObject(context.Background(), bucketName, objectName, 3600*time.Second, nil) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) return @@ -9357,7 +9315,7 @@ func testFunctionalV2() { reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"") // Generate presigned GET object url. args["reqParams"] = reqParams - presignedGetURL, err = c.PresignedGetObject(bucketName, objectName, 3600*time.Second, reqParams) + presignedGetURL, err = c.PresignedGetObject(context.Background(), bucketName, objectName, 3600*time.Second, reqParams) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) return @@ -9402,7 +9360,7 @@ func testFunctionalV2() { "objectName": objectName + "-presigned", "expires": 3600 * time.Second, } - presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second) + presignedPutURL, err := c.PresignedPutObject(context.Background(), bucketName, objectName+"-presigned", 3600*time.Second) if err != nil { logError(testName, function, args, startTime, "", "PresignedPutObject failed", err) return @@ -9429,7 +9387,7 @@ func testFunctionalV2() { "bucketName": bucketName, "objectName": objectName + "-presigned", } - newReader, err = c.GetObject(bucketName, objectName+"-presigned", minio.GetObjectOptions{}) + newReader, err = c.GetObject(context.Background(), bucketName, objectName+"-presigned", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return @@ -9458,12 +9416,12 @@ func testFunctionalV2() { successLogger(testName, functionAll, args, startTime).Info() } -// Test get object with GetObjectWithContext +// Test get object with GetObject with context func testGetObjectWithContext() { // initialize logging params startTime := time.Now() testName := getFuncName() - function := "GetObjectWithContext(ctx, bucketName, objectName)" + function := "GetObject(ctx, bucketName, objectName)" args := map[string]interface{}{ "ctx": "", "bucketName": "", @@ -9495,7 +9453,7 @@ func testGetObjectWithContext() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -9508,7 +9466,7 @@ func testGetObjectWithContext() { objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName - _, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -9518,14 +9476,14 @@ func testGetObjectWithContext() { args["ctx"] = ctx defer cancel() - r, err := c.GetObjectWithContext(ctx, bucketName, objectName, minio.GetObjectOptions{}) + r, err := c.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{}) if err != nil { - logError(testName, function, args, startTime, "", "GetObjectWithContext failed unexpectedly", err) + logError(testName, function, args, startTime, "", "GetObject failed unexpectedly", err) return } if _, err = r.Stat(); err == nil { - logError(testName, function, args, startTime, "", "GetObjectWithContext should fail on short timeout", err) + logError(testName, function, args, startTime, "", "GetObject should fail on short timeout", err) return } r.Close() @@ -9535,9 +9493,9 @@ func testGetObjectWithContext() { defer cancel() // Read the data back - r, err = c.GetObjectWithContext(ctx, bucketName, objectName, minio.GetObjectOptions{}) + r, err = c.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{}) if err != nil { - logError(testName, function, args, startTime, "", "GetObjectWithContext failed", err) + logError(testName, function, args, startTime, "", "GetObject failed", err) return } @@ -9565,12 +9523,12 @@ func testGetObjectWithContext() { } -// Test get object with FGetObjectWithContext +// Test get object with FGetObject with a user provided context func testFGetObjectWithContext() { // initialize logging params startTime := time.Now() testName := getFuncName() - function := "FGetObjectWithContext(ctx, bucketName, objectName, fileName)" + function := "FGetObject(ctx, bucketName, objectName, fileName)" args := map[string]interface{}{ "ctx": "", "bucketName": "", @@ -9603,7 +9561,7 @@ func testFGetObjectWithContext() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -9616,7 +9574,7 @@ func testFGetObjectWithContext() { objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName - _, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -9629,18 +9587,18 @@ func testFGetObjectWithContext() { fileName := "tempfile-context" args["fileName"] = fileName // Read the data back - err = c.FGetObjectWithContext(ctx, bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) + err = c.FGetObject(ctx, bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) if err == nil { - logError(testName, function, args, startTime, "", "FGetObjectWithContext should fail on short timeout", err) + logError(testName, function, args, startTime, "", "FGetObject should fail on short timeout", err) return } ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) defer cancel() // Read the data back - err = c.FGetObjectWithContext(ctx, bucketName, objectName, fileName+"-fcontext", minio.GetObjectOptions{}) + err = c.FGetObject(ctx, bucketName, objectName, fileName+"-fcontext", minio.GetObjectOptions{}) if err != nil { - logError(testName, function, args, startTime, "", "FGetObjectWithContext with long timeout failed", err) + logError(testName, function, args, startTime, "", "FGetObject with long timeout failed", err) return } if err = os.Remove(fileName + "-fcontext"); err != nil { @@ -9657,12 +9615,12 @@ func testFGetObjectWithContext() { } -// Test get object ACLs with GetObjectACLWithContext +// Test get object ACLs with GetObjectACL with custom provided context func testGetObjectACLWithContext() { // initialize logging params startTime := time.Now() testName := getFuncName() - function := "GetObjectACLWithContext(ctx, bucketName, objectName)" + function := "GetObjectACL(ctx, bucketName, objectName)" args := map[string]interface{}{ "ctx": "", "bucketName": "", @@ -9700,7 +9658,7 @@ func testGetObjectACLWithContext() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -9718,7 +9676,7 @@ func testGetObjectACLWithContext() { "X-Amz-Acl": "public-read-write", } - _, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", UserMetadata: metaData}) + _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", UserMetadata: metaData}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -9729,25 +9687,25 @@ func testGetObjectACLWithContext() { defer cancel() // Read the data back - objectInfo, getObjectACLErr := c.GetObjectACLWithContext(ctx, bucketName, objectName) + objectInfo, getObjectACLErr := c.GetObjectACL(ctx, bucketName, objectName) if getObjectACLErr == nil { - logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail", getObjectACLErr) + logError(testName, function, args, startTime, "", "GetObjectACL fail", getObjectACLErr) return } s, ok := objectInfo.Metadata["X-Amz-Acl"] if !ok { - logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail unable to find \"X-Amz-Acl\"", nil) + logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Acl\"", nil) return } if len(s) != 1 { - logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail \"X-Amz-Acl\" canned acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) + logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" canned acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) return } if s[0] != "public-read-write" { - logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail \"X-Amz-Acl\" expected \"public-read-write\" but got"+fmt.Sprintf("%q", s[0]), nil) + logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" expected \"public-read-write\" but got"+fmt.Sprintf("%q", s[0]), nil) return } @@ -9764,7 +9722,7 @@ func testGetObjectACLWithContext() { "X-Amz-Grant-Write": "id=foowrite@minio.go", } - _, err = c.PutObject(bucketName, objectName, reader2, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", UserMetadata: metaData}) + _, err = c.PutObject(context.Background(), bucketName, objectName, reader2, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", UserMetadata: metaData}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -9775,46 +9733,46 @@ func testGetObjectACLWithContext() { defer cancel() // Read the data back - objectInfo, getObjectACLErr = c.GetObjectACLWithContext(ctx, bucketName, objectName) + objectInfo, getObjectACLErr = c.GetObjectACL(ctx, bucketName, objectName) if getObjectACLErr == nil { - logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail", getObjectACLErr) + logError(testName, function, args, startTime, "", "GetObjectACL fail", getObjectACLErr) return } if len(objectInfo.Metadata) != 3 { - logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail expected \"3\" ACLs but got "+fmt.Sprintf(`"%d"`, len(objectInfo.Metadata)), nil) + logError(testName, function, args, startTime, "", "GetObjectACL fail expected \"3\" ACLs but got "+fmt.Sprintf(`"%d"`, len(objectInfo.Metadata)), nil) return } s, ok = objectInfo.Metadata["X-Amz-Grant-Read"] if !ok { - logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail unable to find \"X-Amz-Grant-Read\"", nil) + logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Grant-Read\"", nil) return } if len(s) != 1 { - logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail \"X-Amz-Grant-Read\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) + logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Read\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) return } if s[0] != "fooread@minio.go" { - logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail \"X-Amz-Grant-Read\" acl expected \"fooread@minio.go\" got "+fmt.Sprintf("%q", s), nil) + logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Read\" acl expected \"fooread@minio.go\" got "+fmt.Sprintf("%q", s), nil) return } s, ok = objectInfo.Metadata["X-Amz-Grant-Write"] if !ok { - logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail unable to find \"X-Amz-Grant-Write\"", nil) + logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Grant-Write\"", nil) return } if len(s) != 1 { - logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail \"X-Amz-Grant-Write\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) + logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Write\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) return } if s[0] != "foowrite@minio.go" { - logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail \"X-Amz-Grant-Write\" acl expected \"foowrite@minio.go\" got "+fmt.Sprintf("%q", s), nil) + logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Write\" acl expected \"foowrite@minio.go\" got "+fmt.Sprintf("%q", s), nil) return } @@ -9832,7 +9790,7 @@ func testPutObjectWithContextV2() { // initialize logging params startTime := time.Now() testName := getFuncName() - function := "PutObjectWithContext(ctx, bucketName, objectName, reader, size, opts)" + function := "PutObject(ctx, bucketName, objectName, reader, size, opts)" args := map[string]interface{}{ "ctx": "", "bucketName": "", @@ -9862,12 +9820,12 @@ func testPutObjectWithContextV2() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } - defer c.RemoveBucket(bucketName) + defer c.RemoveBucket(context.Background(), bucketName) bufSize := dataFileMap["datatfile-33-kB"] var reader = getDataReader("datafile-33-kB") defer reader.Close() @@ -9880,9 +9838,9 @@ func testPutObjectWithContextV2() { args["size"] = bufSize defer cancel() - _, err = c.PutObjectWithContext(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + _, err = c.PutObject(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { - logError(testName, function, args, startTime, "", "PutObjectWithContext with short timeout failed", err) + logError(testName, function, args, startTime, "", "PutObject with short timeout failed", err) return } @@ -9892,9 +9850,9 @@ func testPutObjectWithContextV2() { defer cancel() reader = getDataReader("datafile-33-kB") defer reader.Close() - _, err = c.PutObjectWithContext(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + _, err = c.PutObject(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { - logError(testName, function, args, startTime, "", "PutObjectWithContext with long timeout failed", err) + logError(testName, function, args, startTime, "", "PutObject with long timeout failed", err) return } @@ -9908,12 +9866,12 @@ func testPutObjectWithContextV2() { } -// Test get object with GetObjectWithContext +// Test get object with GetObject with custom context func testGetObjectWithContextV2() { // initialize logging params startTime := time.Now() testName := getFuncName() - function := "GetObjectWithContext(ctx, bucketName, objectName)" + function := "GetObject(ctx, bucketName, objectName)" args := map[string]interface{}{ "ctx": "", "bucketName": "", @@ -9945,7 +9903,7 @@ func testGetObjectWithContextV2() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -9958,7 +9916,7 @@ func testGetObjectWithContextV2() { objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName - _, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return @@ -9968,13 +9926,13 @@ func testGetObjectWithContextV2() { args["ctx"] = ctx defer cancel() - r, err := c.GetObjectWithContext(ctx, bucketName, objectName, minio.GetObjectOptions{}) + r, err := c.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{}) if err != nil { - logError(testName, function, args, startTime, "", "GetObjectWithContext failed unexpectedly", err) + logError(testName, function, args, startTime, "", "GetObject failed unexpectedly", err) return } if _, err = r.Stat(); err == nil { - logError(testName, function, args, startTime, "", "GetObjectWithContext should fail on short timeout", err) + logError(testName, function, args, startTime, "", "GetObject should fail on short timeout", err) return } r.Close() @@ -9983,9 +9941,9 @@ func testGetObjectWithContextV2() { defer cancel() // Read the data back - r, err = c.GetObjectWithContext(ctx, bucketName, objectName, minio.GetObjectOptions{}) + r, err = c.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{}) if err != nil { - logError(testName, function, args, startTime, "", "GetObjectWithContext shouldn't fail on longer timeout", err) + logError(testName, function, args, startTime, "", "GetObject shouldn't fail on longer timeout", err) return } @@ -10013,12 +9971,12 @@ func testGetObjectWithContextV2() { } -// Test get object with FGetObjectWithContext +// Test get object with FGetObject with custom context func testFGetObjectWithContextV2() { // initialize logging params startTime := time.Now() testName := getFuncName() - function := "FGetObjectWithContext(ctx, bucketName, objectName,fileName)" + function := "FGetObject(ctx, bucketName, objectName,fileName)" args := map[string]interface{}{ "ctx": "", "bucketName": "", @@ -10051,7 +10009,7 @@ func testFGetObjectWithContextV2() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket call failed", err) return @@ -10064,7 +10022,7 @@ func testFGetObjectWithContextV2() { objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName - _, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return @@ -10078,18 +10036,18 @@ func testFGetObjectWithContextV2() { args["fileName"] = fileName // Read the data back - err = c.FGetObjectWithContext(ctx, bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) + err = c.FGetObject(ctx, bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) if err == nil { - logError(testName, function, args, startTime, "", "FGetObjectWithContext should fail on short timeout", err) + logError(testName, function, args, startTime, "", "FGetObject should fail on short timeout", err) return } ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) defer cancel() // Read the data back - err = c.FGetObjectWithContext(ctx, bucketName, objectName, fileName+"-fcontext", minio.GetObjectOptions{}) + err = c.FGetObject(ctx, bucketName, objectName, fileName+"-fcontext", minio.GetObjectOptions{}) if err != nil { - logError(testName, function, args, startTime, "", "FGetObjectWithContext call shouldn't fail on long timeout", err) + logError(testName, function, args, startTime, "", "FGetObject call shouldn't fail on long timeout", err) return } @@ -10144,7 +10102,7 @@ func testListObjects() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -10165,7 +10123,7 @@ func testListObjects() { bufSize := dataFileMap["datafile-33-kB"] var reader = getDataReader("datafile-33-kB") defer reader.Close() - _, err = c.PutObject(bucketName, object.name, reader, int64(bufSize), + _, err = c.PutObject(context.Background(), bucketName, object.name, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", StorageClass: object.storageClass}) if err != nil { logError(testName, function, args, startTime, "", fmt.Sprintf("PutObject %d call failed", i+1), err) @@ -10173,7 +10131,7 @@ func testListObjects() { } } - testList := func(listFn func(string, string, bool, <-chan struct{}) <-chan minio.ObjectInfo, bucket string) { + testList := func(listFn func(context.Context, string, string, bool, <-chan struct{}) <-chan minio.ObjectInfo, bucket string) { // Create a done channel to control 'ListObjects' go routine. doneCh := make(chan struct{}) // Exit cleanly upon return. @@ -10182,7 +10140,7 @@ func testListObjects() { var objCursor int // check for object name and storage-class from listing object result - for objInfo := range listFn(bucket, "", true, doneCh) { + for objInfo := range listFn(context.Background(), bucket, "", true, doneCh) { if objInfo.Err != nil { logError(testName, function, args, startTime, "", "ListObjects failed unexpectedly", err) return @@ -10253,7 +10211,7 @@ func testRemoveObjectsWithOptions() { args["objectName"] = objectName // Make a new bucket. - err = c.MakeBucketWithObjectLock(bucketName, "us-east-1") + err = c.MakeBucketWithObjectLock(context.Background(), bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -10263,7 +10221,7 @@ func testRemoveObjectsWithOptions() { var reader = getDataReader("datafile-129-MB") defer reader.Close() - n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) + n, err := c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) if err != nil { log.Fatalln(err) } @@ -10276,7 +10234,7 @@ func testRemoveObjectsWithOptions() { RetainUntilDate: &t, Mode: &m, } - err = c.PutObjectRetention(bucketName, objectName, opts) + err = c.PutObjectRetention(context.Background(), bucketName, objectName, opts) if err != nil { log.Fatalln(err) } @@ -10286,7 +10244,7 @@ func testRemoveObjectsWithOptions() { go func() { defer close(objectsCh) // List all objects from a bucket-name with a matching prefix. - for object := range c.ListObjects(bucketName, "", true, nil) { + for object := range c.ListObjects(context.Background(), bucketName, "", true, nil) { if object.Err != nil { log.Fatalln(object.Err) } @@ -10294,7 +10252,7 @@ func testRemoveObjectsWithOptions() { } }() - for rErr := range c.RemoveObjects(bucketName, objectsCh) { + for rErr := range c.RemoveObjects(context.Background(), bucketName, objectsCh) { // Error is expected here because Retention is set on the object // and RemoveObjects is called without Bypass Governance if rErr.Err == nil { @@ -10309,7 +10267,7 @@ func testRemoveObjectsWithOptions() { go func() { defer close(objectsCh1) // List all objects from a bucket-name with a matching prefix. - for object := range c.ListObjects(bucketName, "", true, nil) { + for object := range c.ListObjects(context.Background(), bucketName, "", true, nil) { if object.Err != nil { log.Fatalln(object.Err) } @@ -10321,7 +10279,7 @@ func testRemoveObjectsWithOptions() { GovernanceBypass: true, } - for rErr := range c.RemoveObjectsWithOptions(bucketName, objectsCh1, opts1) { + for rErr := range c.RemoveObjectsWithOptions(context.Background(), bucketName, objectsCh1, opts1) { // Error is not expected here because Retention is set on the object // and RemoveObjects is called with Bypass Governance logError(testName, function, args, startTime, "", "Error detected during deletion", rErr.Err) diff --git a/go.mod b/go.mod index 03497bccf..647c8c2d7 100644 --- a/go.mod +++ b/go.mod @@ -1,16 +1,16 @@ -module github.com/minio/minio-go/v6 +module github.com/minio/minio-go/v7 go 1.12 require ( - github.com/dustin/go-humanize v1.0.0 // indirect github.com/json-iterator/go v1.1.9 github.com/minio/md5-simd v1.1.0 + github.com/minio/minio-go/v6 v6.0.57 // indirect github.com/minio/sha256-simd v0.1.1 github.com/mitchellh/go-homedir v1.1.0 - github.com/sirupsen/logrus v1.5.0 // indirect - github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect + github.com/sirupsen/logrus v1.6.0 // indirect golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f golang.org/x/net v0.0.0-20190522155817-f3200d17e092 + golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae // indirect gopkg.in/ini.v1 v1.42.0 ) diff --git a/go.sum b/go.sum index f158252bb..3a1c62a77 100644 --- a/go.sum +++ b/go.sum @@ -13,10 +13,12 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/minio/md5-simd v1.0.1 h1:tj/FH8APTKxIkOGUX2YGAVJVXXC3AJ5T2SkHoT/dUFI= -github.com/minio/md5-simd v1.0.1/go.mod h1:EhdyA+Dr0guvfyc8d6yrgs9YzHGfaI+YIjA6gt/7mJk= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= +github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= +github.com/minio/minio-go/v6 v6.0.57 h1:ixPkbKkyD7IhnluRgQpGSpHdpvNVaW6OD5R9IAO/9Tw= +github.com/minio/minio-go/v6 v6.0.57/go.mod h1:5+R/nM9Pwrh0vqF+HbYYDQ84wdUFPyXHkrdT4AIkifM= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -27,9 +29,9 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLD github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= -github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q= github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= @@ -48,8 +50,9 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= diff --git a/pkg/credentials/assume_role.go b/pkg/credentials/assume_role.go index f3702d8d6..7456211e7 100644 --- a/pkg/credentials/assume_role.go +++ b/pkg/credentials/assume_role.go @@ -29,7 +29,7 @@ import ( "strings" "time" - "github.com/minio/minio-go/v6/pkg/signer" + "github.com/minio/minio-go/v7/pkg/signer" sha256 "github.com/minio/sha256-simd" ) diff --git a/pkg/policy/bucket-policy-condition.go b/pkg/policy/bucket-policy-condition.go index b256faff4..ae10b589b 100644 --- a/pkg/policy/bucket-policy-condition.go +++ b/pkg/policy/bucket-policy-condition.go @@ -17,7 +17,7 @@ package policy -import "github.com/minio/minio-go/v6/pkg/set" +import "github.com/minio/minio-go/v7/pkg/set" // ConditionKeyMap - map of policy condition key and value. type ConditionKeyMap map[string]set.StringSet diff --git a/pkg/policy/bucket-policy-condition_test.go b/pkg/policy/bucket-policy-condition_test.go index db24cf364..1fc154868 100644 --- a/pkg/policy/bucket-policy-condition_test.go +++ b/pkg/policy/bucket-policy-condition_test.go @@ -21,7 +21,7 @@ import ( "testing" jsoniter "github.com/json-iterator/go" - "github.com/minio/minio-go/v6/pkg/set" + "github.com/minio/minio-go/v7/pkg/set" ) var json = jsoniter.ConfigCompatibleWithStandardLibrary diff --git a/pkg/policy/bucket-policy.go b/pkg/policy/bucket-policy.go index f318be2a0..bb8c51b07 100644 --- a/pkg/policy/bucket-policy.go +++ b/pkg/policy/bucket-policy.go @@ -23,7 +23,7 @@ import ( "strings" jsoniter "github.com/json-iterator/go" - "github.com/minio/minio-go/v6/pkg/set" + "github.com/minio/minio-go/v7/pkg/set" ) // BucketPolicy - Bucket level policy. diff --git a/pkg/policy/bucket-policy_test.go b/pkg/policy/bucket-policy_test.go index 7d0a9cbfb..6848dbd01 100644 --- a/pkg/policy/bucket-policy_test.go +++ b/pkg/policy/bucket-policy_test.go @@ -22,7 +22,7 @@ import ( "reflect" "testing" - "github.com/minio/minio-go/v6/pkg/set" + "github.com/minio/minio-go/v7/pkg/set" ) // TestUnmarshalBucketPolicy tests unmarsheling various examples diff --git a/pkg/signer/request-signature-v2.go b/pkg/signer/request-signature-v2.go index 3179d62b5..d908f2779 100644 --- a/pkg/signer/request-signature-v2.go +++ b/pkg/signer/request-signature-v2.go @@ -30,7 +30,7 @@ import ( "strings" "time" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) // Signature and API related constants. diff --git a/pkg/signer/request-signature-v4.go b/pkg/signer/request-signature-v4.go index 0f6a4c0a6..67572b20d 100644 --- a/pkg/signer/request-signature-v4.go +++ b/pkg/signer/request-signature-v4.go @@ -26,7 +26,7 @@ import ( "strings" "time" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) // Signature and API related constants. diff --git a/utils.go b/utils.go index 0214644e4..b5cca44a2 100644 --- a/utils.go +++ b/utils.go @@ -35,7 +35,7 @@ import ( "time" md5simd "github.com/minio/md5-simd" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" "github.com/minio/sha256-simd" ) diff --git a/utils_test.go b/utils_test.go index 70f80fa91..856d7319a 100644 --- a/utils_test.go +++ b/utils_test.go @@ -23,7 +23,7 @@ import ( "testing" "time" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) // Tests signature redacting function used From 5a587ea5f1eaec3bf64f796177d23b55d324454f Mon Sep 17 00:00:00 2001 From: poornas Date: Thu, 25 Jun 2020 18:32:18 -0700 Subject: [PATCH 166/215] Add x-amz-replication-status to supported headers (#1306) --- api-put-object-streaming.go | 7 ++++++- api-put-object.go | 5 +++++ constants.go | 3 +++ utils.go | 1 + 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/api-put-object-streaming.go b/api-put-object-streaming.go index 2fedc6584..9134abdfa 100644 --- a/api-put-object-streaming.go +++ b/api-put-object-streaming.go @@ -24,6 +24,7 @@ import ( "fmt" "io" "net/http" + "net/url" "sort" "strings" @@ -437,7 +438,11 @@ func (c Client) putObjectDo(ctx context.Context, bucketName, objectName string, contentMD5Base64: md5Base64, contentSHA256Hex: sha256Hex, } - + if opts.ReplicationVersionID != "" { + urlValues := make(url.Values) + urlValues.Set("versionId", opts.ReplicationVersionID) + reqMetadata.queryValues = urlValues + } // Execute PUT an objectName. resp, err := c.executeMethod(ctx, "PUT", reqMetadata) defer closeResponse(resp) diff --git a/api-put-object.go b/api-put-object.go index 5518758b7..a9a2b4512 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -53,6 +53,8 @@ type PutObjectOptions struct { LegalHold LegalHoldStatus SendContentMd5 bool DisableMultipart bool + ReplicationVersionID string + ReplicationStatus string } // getNumThreads - gets the number of threads to be used in the multipart @@ -114,6 +116,9 @@ func (opts PutObjectOptions) Header() (header http.Header) { header.Set(amzWebsiteRedirectLocation, opts.WebsiteRedirectLocation) } + if opts.ReplicationStatus != "" { + header.Set(amzBucketReplicationStatus, opts.ReplicationStatus) + } if len(opts.UserTags) != 0 { header.Set(amzTaggingHeader, s3utils.TagEncode(opts.UserTags)) } diff --git a/constants.go b/constants.go index 78ba5a114..af4d44c8f 100644 --- a/constants.go +++ b/constants.go @@ -73,4 +73,7 @@ const ( amzLockMode = "X-Amz-Object-Lock-Mode" amzLockRetainUntil = "X-Amz-Object-Lock-Retain-Until-Date" amzBypassGovernance = "X-Amz-Bypass-Governance-Retention" + + // Replication status + amzBucketReplicationStatus = "X-Amz-Replication-Status" ) diff --git a/utils.go b/utils.go index b5cca44a2..ab20028b6 100644 --- a/utils.go +++ b/utils.go @@ -332,6 +332,7 @@ var supportedHeaders = []string{ "x-amz-metadata-directive", "x-amz-object-lock-retain-until-date", "expires", + "x-amz-replication-status", // Add more supported headers here. } From 48937efa2a4f78a0c56ac850187a31068cb2ac52 Mon Sep 17 00:00:00 2001 From: poornas Date: Fri, 26 Jun 2020 12:04:29 -0700 Subject: [PATCH 167/215] fix docs for replication opts in API.md (#1312) --- docs/API.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/API.md b/docs/API.md index efe2aa4b8..d8c180647 100644 --- a/docs/API.md +++ b/docs/API.md @@ -622,6 +622,8 @@ __minio.PutObjectOptions__ | `opts.WebsiteRedirectLocation` | _string_ | Specify a redirect for the object, to another object in the same bucket or to a external URL. | | `opts.SendContentMd5` | _bool_ | Specify if you'd like to send `content-md5` header with PutObject operation. Note that setting this flag will cause higher memory usage because of in-memory `md5sum` calculation. | | `opts.PartSize` | _uint64_ | Specify a custom part size used for uploading the object | +| `opts.ReplicationVersionID` | _string_ | Specify VersionID of object to replicate.This option is intended for internal use by MinIO server to extend the replication API implementation by AWS. This option should not be set unless the application is aware of intended use. | +| `opts.ReplicationStatus` | _string_ | Specify replication status of object. This option is intended for internal use by MinIO server to extend the replication API implementation by AWS. This option should not be set unless the application is aware of intended use. | __Example__ From a1a842a6a094537e81f03ac36130e02c372c7b55 Mon Sep 17 00:00:00 2001 From: kannappanr <30541348+kannappanr@users.noreply.github.com> Date: Tue, 30 Jun 2020 20:14:39 -0700 Subject: [PATCH 168/215] Allow empty retention to be set (#1320) --- api-object-retention.go | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/api-object-retention.go b/api-object-retention.go index 24c31af0d..0b85b6d27 100644 --- a/api-object-retention.go +++ b/api-object-retention.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2019 MinIO, Inc. + * Copyright 2019-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -34,26 +34,23 @@ import ( type objectRetention struct { XMLNS string `xml:"xmlns,attr,omitempty"` XMLName xml.Name `xml:"Retention"` - Mode RetentionMode `xml:"Mode"` - RetainUntilDate time.Time `type:"timestamp" timestampFormat:"iso8601" xml:"RetainUntilDate"` + Mode RetentionMode `xml:"Mode,omitempty"` + RetainUntilDate *time.Time `type:"timestamp" timestampFormat:"iso8601" xml:"RetainUntilDate,omitempty"` } func newObjectRetention(mode *RetentionMode, date *time.Time) (*objectRetention, error) { - if mode == nil { - return nil, fmt.Errorf("Mode not set") - } - - if date == nil { - return nil, fmt.Errorf("RetainUntilDate not set") - } + objectRetention := &objectRetention{} - if !mode.IsValid() { - return nil, fmt.Errorf("invalid retention mode `%v`", mode) + if date != nil && !date.IsZero() { + objectRetention.RetainUntilDate = date } - objectRetention := &objectRetention{ - Mode: *mode, - RetainUntilDate: *date, + if mode != nil { + if !mode.IsValid() { + return nil, fmt.Errorf("invalid retention mode `%v`", mode) + } + objectRetention.Mode = *mode } + return objectRetention, nil } @@ -164,5 +161,5 @@ func (c Client) GetObjectRetention(ctx context.Context, bucketName, objectName, return nil, nil, err } - return &retention.Mode, &retention.RetainUntilDate, nil + return &retention.Mode, retention.RetainUntilDate, nil } From b76e654429571a149042b580e5e7d78a757af4d4 Mon Sep 17 00:00:00 2001 From: Anis Elleuch Date: Wed, 1 Jul 2020 20:05:15 +0100 Subject: [PATCH 169/215] Unexport some ErrorResponse generator functions (#1317) --- api-compose-object.go | 28 ++++++++++---------- api-error-response.go | 51 +++++++++++++------------------------ api-error-response_test.go | 12 ++++----- api-get-object-file.go | 2 +- api-get-object.go | 18 ++++++------- api-get-options.go | 10 ++++---- api-notification.go | 2 +- api-presigned.go | 4 +-- api-put-object-common.go | 10 ++++---- api-put-object-multipart.go | 12 ++++----- api-put-object-streaming.go | 16 ++++++------ api-put-object.go | 12 ++++----- api-remove.go | 4 +-- api.go | 6 ++--- post-policy.go | 32 +++++++++++------------ utils.go | 16 ++++++------ utils_test.go | 38 +++++++++++++-------------- 17 files changed, 129 insertions(+), 144 deletions(-) diff --git a/api-compose-object.go b/api-compose-object.go index 10df794eb..ec8c05577 100644 --- a/api-compose-object.go +++ b/api-compose-object.go @@ -79,7 +79,7 @@ func filterCustomMeta(userMeta map[string]string) (map[string]string, error) { k = k[len("x-amz-meta-"):] } if _, ok := m[k]; ok { - return nil, ErrInvalidArgument(fmt.Sprintf("Cannot add both %s and x-amz-meta-%s keys as custom metadata", k, k)) + return nil, errInvalidArgument(fmt.Sprintf("Cannot add both %s and x-amz-meta-%s keys as custom metadata", k, k)) } m[k] = v } @@ -205,7 +205,7 @@ func NewSourceInfo(bucket, object string, sse encrypt.ServerSide) SourceInfo { // copied. func (s *SourceInfo) SetRange(start, end int64) error { if start > end || start < 0 { - return ErrInvalidArgument("start must be non-negative, and start must be at most end.") + return errInvalidArgument("start must be non-negative, and start must be at most end.") } // Note that 0 <= start <= end s.start, s.end = start, end @@ -216,7 +216,7 @@ func (s *SourceInfo) SetRange(start, end int64) error { // only if the etag of the source matches the value given here. func (s *SourceInfo) SetMatchETagCond(etag string) error { if etag == "" { - return ErrInvalidArgument("ETag cannot be empty.") + return errInvalidArgument("ETag cannot be empty.") } s.Headers.Set("x-amz-copy-source-if-match", etag) return nil @@ -227,7 +227,7 @@ func (s *SourceInfo) SetMatchETagCond(etag string) error { // not the value given here. func (s *SourceInfo) SetMatchETagExceptCond(etag string) error { if etag == "" { - return ErrInvalidArgument("ETag cannot be empty.") + return errInvalidArgument("ETag cannot be empty.") } s.Headers.Set("x-amz-copy-source-if-none-match", etag) return nil @@ -236,7 +236,7 @@ func (s *SourceInfo) SetMatchETagExceptCond(etag string) error { // SetModifiedSinceCond - Set the modified since condition. func (s *SourceInfo) SetModifiedSinceCond(modTime time.Time) error { if modTime.IsZero() { - return ErrInvalidArgument("Input time cannot be 0.") + return errInvalidArgument("Input time cannot be 0.") } s.Headers.Set("x-amz-copy-source-if-modified-since", modTime.Format(http.TimeFormat)) return nil @@ -245,7 +245,7 @@ func (s *SourceInfo) SetModifiedSinceCond(modTime time.Time) error { // SetUnmodifiedSinceCond - Set the unmodified since condition. func (s *SourceInfo) SetUnmodifiedSinceCond(modTime time.Time) error { if modTime.IsZero() { - return ErrInvalidArgument("Input time cannot be 0.") + return errInvalidArgument("Input time cannot be 0.") } s.Headers.Set("x-amz-copy-source-if-unmodified-since", modTime.Format(http.TimeFormat)) return nil @@ -259,7 +259,7 @@ func (s *SourceInfo) getProps(c Client) (size int64, etag string, userMeta map[s opts := StatObjectOptions{GetObjectOptions{ServerSideEncryption: encrypt.SSE(s.encryption)}} objInfo, err = c.statObject(context.Background(), s.bucket, s.object, opts) if err != nil { - err = ErrInvalidArgument(fmt.Sprintf("Could not stat object - %s/%s: %v", s.bucket, s.object, err)) + err = errInvalidArgument(fmt.Sprintf("Could not stat object - %s/%s: %v", s.bucket, s.object, err)) } else { size = objInfo.Size etag = objInfo.ETag @@ -329,7 +329,7 @@ func (c Client) copyObjectPartDo(ctx context.Context, srcBucket, srcObject, dest headers.Set("x-amz-copy-source", s3utils.EncodePath(srcBucket+"/"+srcObject)) if startOffset < 0 { - return p, ErrInvalidArgument("startOffset must be non-negative") + return p, errInvalidArgument("startOffset must be non-negative") } if length >= 0 { @@ -415,7 +415,7 @@ func (c Client) uploadPartCopy(ctx context.Context, bucket, object, uploadID str // look at current progress. func (c Client) ComposeObjectWithProgress(ctx context.Context, dst DestinationInfo, srcs []SourceInfo, progress io.Reader) error { if len(srcs) < 1 || len(srcs) > maxPartsCount { - return ErrInvalidArgument("There must be as least one and up to 10000 source objects.") + return errInvalidArgument("There must be as least one and up to 10000 source objects.") } srcSizes := make([]int64, len(srcs)) var totalSize, size, totalParts int64 @@ -431,7 +431,7 @@ func (c Client) ComposeObjectWithProgress(ctx context.Context, dst DestinationIn // Error out if client side encryption is used in this source object when // more than one source objects are given. if len(srcs) > 1 && src.Headers.Get("x-amz-meta-x-amz-key") != "" { - return ErrInvalidArgument( + return errInvalidArgument( fmt.Sprintf("Client side encryption is used in source object %s/%s", src.bucket, src.object)) } @@ -442,7 +442,7 @@ func (c Client) ComposeObjectWithProgress(ctx context.Context, dst DestinationIn // 0 <= src.start <= src.end // so only invalid case to check is: if src.end >= size { - return ErrInvalidArgument( + return errInvalidArgument( fmt.Sprintf("SourceInfo %d has invalid segment-to-copy [%d, %d] (size is %d)", i, src.start, src.end, size)) } @@ -451,14 +451,14 @@ func (c Client) ComposeObjectWithProgress(ctx context.Context, dst DestinationIn // Only the last source may be less than `absMinPartSize` if size < absMinPartSize && i < len(srcs)-1 { - return ErrInvalidArgument( + return errInvalidArgument( fmt.Sprintf("SourceInfo %d is too small (%d) and it is not the last part", i, size)) } // Is data to copy too large? totalSize += size if totalSize > maxMultipartPutObjectSize { - return ErrInvalidArgument(fmt.Sprintf("Cannot compose an object of size %d (> 5TiB)", totalSize)) + return errInvalidArgument(fmt.Sprintf("Cannot compose an object of size %d (> 5TiB)", totalSize)) } // record source size @@ -468,7 +468,7 @@ func (c Client) ComposeObjectWithProgress(ctx context.Context, dst DestinationIn totalParts += partsRequired(size) // Do we need more parts than we are allowed? if totalParts > maxPartsCount { - return ErrInvalidArgument(fmt.Sprintf( + return errInvalidArgument(fmt.Sprintf( "Your proposed compose object requires more than %d parts", maxPartsCount)) } } diff --git a/api-error-response.go b/api-error-response.go index 4628f2c3e..f439a8870 100644 --- a/api-error-response.go +++ b/api-error-response.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -103,7 +103,7 @@ const ( func httpRespToErrorResponse(resp *http.Response, bucketName, objectName string) error { if resp == nil { msg := "Response is empty. " + reportIssue - return ErrInvalidArgument(msg) + return errInvalidArgument(msg) } errResp := ErrorResponse{ @@ -183,8 +183,8 @@ func httpRespToErrorResponse(resp *http.Response, bucketName, objectName string) return errResp } -// ErrTransferAccelerationBucket - bucket name is invalid to be used with transfer acceleration. -func ErrTransferAccelerationBucket(bucketName string) error { +// errTransferAccelerationBucket - bucket name is invalid to be used with transfer acceleration. +func errTransferAccelerationBucket(bucketName string) error { return ErrorResponse{ StatusCode: http.StatusBadRequest, Code: "InvalidArgument", @@ -193,8 +193,8 @@ func ErrTransferAccelerationBucket(bucketName string) error { } } -// ErrEntityTooLarge - Input size is larger than supported maximum. -func ErrEntityTooLarge(totalSize, maxObjectSize int64, bucketName, objectName string) error { +// errEntityTooLarge - Input size is larger than supported maximum. +func errEntityTooLarge(totalSize, maxObjectSize int64, bucketName, objectName string) error { msg := fmt.Sprintf("Your proposed upload size ‘%d’ exceeds the maximum allowed object size ‘%d’ for single PUT operation.", totalSize, maxObjectSize) return ErrorResponse{ StatusCode: http.StatusBadRequest, @@ -205,8 +205,8 @@ func ErrEntityTooLarge(totalSize, maxObjectSize int64, bucketName, objectName st } } -// ErrEntityTooSmall - Input size is smaller than supported minimum. -func ErrEntityTooSmall(totalSize int64, bucketName, objectName string) error { +// errEntityTooSmall - Input size is smaller than supported minimum. +func errEntityTooSmall(totalSize int64, bucketName, objectName string) error { msg := fmt.Sprintf("Your proposed upload size ‘%d’ is below the minimum allowed object size ‘0B’ for single PUT operation.", totalSize) return ErrorResponse{ StatusCode: http.StatusBadRequest, @@ -217,8 +217,8 @@ func ErrEntityTooSmall(totalSize int64, bucketName, objectName string) error { } } -// ErrUnexpectedEOF - Unexpected end of file reached. -func ErrUnexpectedEOF(totalRead, totalSize int64, bucketName, objectName string) error { +// errUnexpectedEOF - Unexpected end of file reached. +func errUnexpectedEOF(totalRead, totalSize int64, bucketName, objectName string) error { msg := fmt.Sprintf("Data read ‘%d’ is not equal to the size ‘%d’ of the input Reader.", totalRead, totalSize) return ErrorResponse{ StatusCode: http.StatusBadRequest, @@ -229,8 +229,8 @@ func ErrUnexpectedEOF(totalRead, totalSize int64, bucketName, objectName string) } } -// ErrInvalidBucketName - Invalid bucket name response. -func ErrInvalidBucketName(message string) error { +// errInvalidBucketName - Invalid bucket name response. +func errInvalidBucketName(message string) error { return ErrorResponse{ StatusCode: http.StatusBadRequest, Code: "InvalidBucketName", @@ -239,8 +239,8 @@ func ErrInvalidBucketName(message string) error { } } -// ErrInvalidObjectName - Invalid object name response. -func ErrInvalidObjectName(message string) error { +// errInvalidObjectName - Invalid object name response. +func errInvalidObjectName(message string) error { return ErrorResponse{ StatusCode: http.StatusNotFound, Code: "NoSuchKey", @@ -249,12 +249,8 @@ func ErrInvalidObjectName(message string) error { } } -// ErrInvalidObjectPrefix - Invalid object prefix response is -// similar to object name response. -var ErrInvalidObjectPrefix = ErrInvalidObjectName - -// ErrInvalidArgument - Invalid argument response. -func ErrInvalidArgument(message string) error { +// errInvalidArgument - Invalid argument response. +func errInvalidArgument(message string) error { return ErrorResponse{ StatusCode: http.StatusBadRequest, Code: "InvalidArgument", @@ -263,20 +259,9 @@ func ErrInvalidArgument(message string) error { } } -// ErrNoSuchBucketPolicy - No Such Bucket Policy response -// The specified bucket does not have a bucket policy. -func ErrNoSuchBucketPolicy(message string) error { - return ErrorResponse{ - StatusCode: http.StatusNotFound, - Code: "NoSuchBucketPolicy", - Message: message, - RequestID: "minio", - } -} - -// ErrAPINotSupported - API not supported response +// errAPINotSupported - API not supported response // The specified API call is not supported -func ErrAPINotSupported(message string) error { +func errAPINotSupported(message string) error { return ErrorResponse{ StatusCode: http.StatusNotImplemented, Code: "APINotSupported", diff --git a/api-error-response_test.go b/api-error-response_test.go index 0111bd904..8e823183b 100644 --- a/api-error-response_test.go +++ b/api-error-response_test.go @@ -193,7 +193,7 @@ func TestErrEntityTooLarge(t *testing.T) { BucketName: "minio-bucket", Key: "Asia/", } - actualResult := ErrEntityTooLarge(1000000, 99999, "minio-bucket", "Asia/") + actualResult := errEntityTooLarge(1000000, 99999, "minio-bucket", "Asia/") if !reflect.DeepEqual(expectedResult, actualResult) { t.Errorf("Expected result to be '%#v', but instead got '%#v'", expectedResult, actualResult) } @@ -209,7 +209,7 @@ func TestErrEntityTooSmall(t *testing.T) { BucketName: "minio-bucket", Key: "Asia/", } - actualResult := ErrEntityTooSmall(-1, "minio-bucket", "Asia/") + actualResult := errEntityTooSmall(-1, "minio-bucket", "Asia/") if !reflect.DeepEqual(expectedResult, actualResult) { t.Errorf("Expected result to be '%#v', but instead got '%#v'", expectedResult, actualResult) } @@ -226,7 +226,7 @@ func TestErrUnexpectedEOF(t *testing.T) { BucketName: "minio-bucket", Key: "Asia/", } - actualResult := ErrUnexpectedEOF(100, 101, "minio-bucket", "Asia/") + actualResult := errUnexpectedEOF(100, 101, "minio-bucket", "Asia/") if !reflect.DeepEqual(expectedResult, actualResult) { t.Errorf("Expected result to be '%#v', but instead got '%#v'", expectedResult, actualResult) } @@ -240,7 +240,7 @@ func TestErrInvalidBucketName(t *testing.T) { Message: "Invalid Bucket name", RequestID: "minio", } - actualResult := ErrInvalidBucketName("Invalid Bucket name") + actualResult := errInvalidBucketName("Invalid Bucket name") if !reflect.DeepEqual(expectedResult, actualResult) { t.Errorf("Expected result to be '%#v', but instead got '%#v'", expectedResult, actualResult) } @@ -254,7 +254,7 @@ func TestErrInvalidObjectName(t *testing.T) { Message: "Invalid Object Key", RequestID: "minio", } - actualResult := ErrInvalidObjectName("Invalid Object Key") + actualResult := errInvalidObjectName("Invalid Object Key") if !reflect.DeepEqual(expectedResult, actualResult) { t.Errorf("Expected result to be '%#v', but instead got '%#v'", expectedResult, actualResult) } @@ -268,7 +268,7 @@ func TestErrInvalidArgument(t *testing.T) { Message: "Invalid Argument", RequestID: "minio", } - actualResult := ErrInvalidArgument("Invalid Argument") + actualResult := errInvalidArgument("Invalid Argument") if !reflect.DeepEqual(expectedResult, actualResult) { t.Errorf("Expected result to be '%#v', but instead got '%#v'", expectedResult, actualResult) } diff --git a/api-get-object-file.go b/api-get-object-file.go index 464854ae3..25f8d15c5 100644 --- a/api-get-object-file.go +++ b/api-get-object-file.go @@ -42,7 +42,7 @@ func (c Client) FGetObject(ctx context.Context, bucketName, objectName, filePath if err == nil { // If the destination exists and is a directory. if st.IsDir() { - return ErrInvalidArgument("fileName is a directory.") + return errInvalidArgument("fileName is a directory.") } } diff --git a/api-get-object.go b/api-get-object.go index e88a85e54..a3218f3b4 100644 --- a/api-get-object.go +++ b/api-get-object.go @@ -317,7 +317,7 @@ func (o *Object) setOffset(bytesRead int64) error { // io.EOF upon end of file. func (o *Object) Read(b []byte) (n int, err error) { if o == nil { - return 0, ErrInvalidArgument("Object is nil") + return 0, errInvalidArgument("Object is nil") } // Locking. @@ -371,7 +371,7 @@ func (o *Object) Read(b []byte) (n int, err error) { // Stat returns the ObjectInfo structure describing Object. func (o *Object) Stat() (ObjectInfo, error) { if o == nil { - return ObjectInfo{}, ErrInvalidArgument("Object is nil") + return ObjectInfo{}, errInvalidArgument("Object is nil") } // Locking. o.mutex.Lock() @@ -403,7 +403,7 @@ func (o *Object) Stat() (ObjectInfo, error) { // file, that error is io.EOF. func (o *Object) ReadAt(b []byte, offset int64) (n int, err error) { if o == nil { - return 0, ErrInvalidArgument("Object is nil") + return 0, errInvalidArgument("Object is nil") } // Locking. @@ -480,7 +480,7 @@ func (o *Object) ReadAt(b []byte, offset int64) (n int, err error) { // underlying object is not closed. func (o *Object) Seek(offset int64, whence int) (n int64, err error) { if o == nil { - return 0, ErrInvalidArgument("Object is nil") + return 0, errInvalidArgument("Object is nil") } // Locking. @@ -494,7 +494,7 @@ func (o *Object) Seek(offset int64, whence int) (n int64, err error) { // Negative offset is valid for whence of '2'. if offset < 0 && whence != 2 { - return 0, ErrInvalidArgument(fmt.Sprintf("Negative position not allowed for %d", whence)) + return 0, errInvalidArgument(fmt.Sprintf("Negative position not allowed for %d", whence)) } // This is the first request. So before anything else @@ -518,7 +518,7 @@ func (o *Object) Seek(offset int64, whence int) (n int64, err error) { // Switch through whence. switch whence { default: - return 0, ErrInvalidArgument(fmt.Sprintf("Invalid whence %d", whence)) + return 0, errInvalidArgument(fmt.Sprintf("Invalid whence %d", whence)) case 0: if o.objectInfo.Size > -1 && offset > o.objectInfo.Size { return 0, io.EOF @@ -532,7 +532,7 @@ func (o *Object) Seek(offset int64, whence int) (n int64, err error) { case 2: // If we don't know the object size return an error for io.SeekEnd if o.objectInfo.Size < 0 { - return 0, ErrInvalidArgument("Whence END is not supported when the object size is unknown") + return 0, errInvalidArgument("Whence END is not supported when the object size is unknown") } // Seeking to positive offset is valid for whence '2', but // since we are backing a Reader we have reached 'EOF' if @@ -542,7 +542,7 @@ func (o *Object) Seek(offset int64, whence int) (n int64, err error) { } // Seeking to negative position not allowed for whence. if o.objectInfo.Size+offset < 0 { - return 0, ErrInvalidArgument(fmt.Sprintf("Seeking at negative offset not allowed for %d", whence)) + return 0, errInvalidArgument(fmt.Sprintf("Seeking at negative offset not allowed for %d", whence)) } o.currOffset = o.objectInfo.Size + offset } @@ -563,7 +563,7 @@ func (o *Object) Seek(offset int64, whence int) (n int64, err error) { // for subsequent Close() calls. func (o *Object) Close() (err error) { if o == nil { - return ErrInvalidArgument("Object is nil") + return errInvalidArgument("Object is nil") } // Locking. o.mutex.Lock() diff --git a/api-get-options.go b/api-get-options.go index ff38ba72b..b169186e3 100644 --- a/api-get-options.go +++ b/api-get-options.go @@ -63,7 +63,7 @@ func (o *GetObjectOptions) Set(key, value string) { // SetMatchETag - set match etag. func (o *GetObjectOptions) SetMatchETag(etag string) error { if etag == "" { - return ErrInvalidArgument("ETag cannot be empty.") + return errInvalidArgument("ETag cannot be empty.") } o.Set("If-Match", "\""+etag+"\"") return nil @@ -72,7 +72,7 @@ func (o *GetObjectOptions) SetMatchETag(etag string) error { // SetMatchETagExcept - set match etag except. func (o *GetObjectOptions) SetMatchETagExcept(etag string) error { if etag == "" { - return ErrInvalidArgument("ETag cannot be empty.") + return errInvalidArgument("ETag cannot be empty.") } o.Set("If-None-Match", "\""+etag+"\"") return nil @@ -81,7 +81,7 @@ func (o *GetObjectOptions) SetMatchETagExcept(etag string) error { // SetUnmodified - set unmodified time since. func (o *GetObjectOptions) SetUnmodified(modTime time.Time) error { if modTime.IsZero() { - return ErrInvalidArgument("Modified since cannot be empty.") + return errInvalidArgument("Modified since cannot be empty.") } o.Set("If-Unmodified-Since", modTime.Format(http.TimeFormat)) return nil @@ -90,7 +90,7 @@ func (o *GetObjectOptions) SetUnmodified(modTime time.Time) error { // SetModified - set modified time since. func (o *GetObjectOptions) SetModified(modTime time.Time) error { if modTime.IsZero() { - return ErrInvalidArgument("Modified since cannot be empty.") + return errInvalidArgument("Modified since cannot be empty.") } o.Set("If-Modified-Since", modTime.Format(http.TimeFormat)) return nil @@ -119,7 +119,7 @@ func (o *GetObjectOptions) SetRange(start, end int64) error { // bytes=-3-0 // bytes=-3--2 // are invalid. - return ErrInvalidArgument( + return errInvalidArgument( fmt.Sprintf( "Invalid range specified: start=%d end=%d", start, end)) diff --git a/api-notification.go b/api-notification.go index 0d91d7800..9d3b16452 100644 --- a/api-notification.go +++ b/api-notification.go @@ -159,7 +159,7 @@ func (c Client) ListenBucketNotification(ctx context.Context, bucketName, prefix if s3utils.IsAmazonEndpoint(*c.endpointURL) || s3utils.IsGoogleEndpoint(*c.endpointURL) { select { case notificationInfoCh <- NotificationInfo{ - Err: ErrAPINotSupported("Listening for bucket notification is specific only to `minio` server endpoints"), + Err: errAPINotSupported("Listening for bucket notification is specific only to `minio` server endpoints"), }: case <-doneCh: } diff --git a/api-presigned.go b/api-presigned.go index 6e8958dd2..c06a4fe7d 100644 --- a/api-presigned.go +++ b/api-presigned.go @@ -33,7 +33,7 @@ import ( func (c Client) presignURL(ctx context.Context, method string, bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { // Input validation. if method == "" { - return nil, ErrInvalidArgument("method cannot be empty.") + return nil, errInvalidArgument("method cannot be empty.") } if err = s3utils.CheckValidBucketName(bucketName); err != nil { return nil, err @@ -141,7 +141,7 @@ func (c Client) PresignedPostPolicy(ctx context.Context, p *PostPolicy) (u *url. ) if signerType.IsAnonymous() { - return nil, nil, ErrInvalidArgument("Presigned operations are not supported for anonymous credentials") + return nil, nil, errInvalidArgument("Presigned operations are not supported for anonymous credentials") } // Keep time. diff --git a/api-put-object-common.go b/api-put-object-common.go index d3ff8d023..3d0408e53 100644 --- a/api-put-object-common.go +++ b/api-put-object-common.go @@ -77,31 +77,31 @@ func optimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCou // object size is larger than supported maximum. if objectSize > maxMultipartPutObjectSize { - err = ErrEntityTooLarge(objectSize, maxMultipartPutObjectSize, "", "") + err = errEntityTooLarge(objectSize, maxMultipartPutObjectSize, "", "") return } var partSizeFlt float64 if configuredPartSize > 0 { if int64(configuredPartSize) > objectSize { - err = ErrEntityTooLarge(int64(configuredPartSize), objectSize, "", "") + err = errEntityTooLarge(int64(configuredPartSize), objectSize, "", "") return } if !unknownSize { if objectSize > (int64(configuredPartSize) * maxPartsCount) { - err = ErrInvalidArgument("Part size * max_parts(10000) is lesser than input objectSize.") + err = errInvalidArgument("Part size * max_parts(10000) is lesser than input objectSize.") return } } if configuredPartSize < absMinPartSize { - err = ErrInvalidArgument("Input part size is smaller than allowed minimum of 5MiB.") + err = errInvalidArgument("Input part size is smaller than allowed minimum of 5MiB.") return } if configuredPartSize > maxPartSize { - err = ErrInvalidArgument("Input part size is bigger than allowed maximum of 5GiB.") + err = errInvalidArgument("Input part size is bigger than allowed maximum of 5GiB.") return } diff --git a/api-put-object-multipart.go b/api-put-object-multipart.go index c80cdae68..29c7f1f46 100644 --- a/api-put-object-multipart.go +++ b/api-put-object-multipart.go @@ -46,7 +46,7 @@ func (c Client) putObjectMultipart(ctx context.Context, bucketName, objectName s if errResp.Code == "AccessDenied" && strings.Contains(errResp.Message, "Access Denied") { // Verify if size of reader is greater than '5GiB'. if size > maxSinglePutObjectSize { - return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) + return 0, errEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) } // Fall back to uploading as single PutObject operation. return c.putObject(ctx, bucketName, objectName, reader, size, opts) @@ -163,7 +163,7 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje for i := 1; i < partNumber; i++ { part, ok := partsInfo[i] if !ok { - return 0, ErrInvalidArgument(fmt.Sprintf("Missing part number %d", i)) + return 0, errInvalidArgument(fmt.Sprintf("Missing part number %d", i)) } complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ ETag: part.ETag, @@ -236,16 +236,16 @@ func (c Client) uploadPart(ctx context.Context, bucketName, objectName, uploadID return ObjectPart{}, err } if size > maxPartSize { - return ObjectPart{}, ErrEntityTooLarge(size, maxPartSize, bucketName, objectName) + return ObjectPart{}, errEntityTooLarge(size, maxPartSize, bucketName, objectName) } if size <= -1 { - return ObjectPart{}, ErrEntityTooSmall(size, bucketName, objectName) + return ObjectPart{}, errEntityTooSmall(size, bucketName, objectName) } if partNumber <= 0 { - return ObjectPart{}, ErrInvalidArgument("Part number cannot be negative or equal to zero.") + return ObjectPart{}, errInvalidArgument("Part number cannot be negative or equal to zero.") } if uploadID == "" { - return ObjectPart{}, ErrInvalidArgument("UploadID cannot be empty.") + return ObjectPart{}, errInvalidArgument("UploadID cannot be empty.") } // Get resources properly escaped and lined up before using them in http request. diff --git a/api-put-object-streaming.go b/api-put-object-streaming.go index 9134abdfa..657fd9fb3 100644 --- a/api-put-object-streaming.go +++ b/api-put-object-streaming.go @@ -56,7 +56,7 @@ func (c Client) putObjectMultipartStream(ctx context.Context, bucketName, object if errResp.Code == "AccessDenied" && strings.Contains(errResp.Message, "Access Denied") { // Verify if size of reader is greater than '5GiB'. if size > maxSinglePutObjectSize { - return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) + return 0, errEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) } // Fall back to uploading as single PutObject operation. return c.putObject(ctx, bucketName, objectName, reader, size, opts) @@ -209,7 +209,7 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa // Verify if we uploaded all the data. if totalUploadedSize != size { - return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) + return totalUploadedSize, errUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) } // Sort all completed parts. @@ -316,7 +316,7 @@ func (c Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, bu // Verify if we uploaded all the data. if size > 0 { if totalUploadedSize != size { - return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) + return totalUploadedSize, errUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) } } @@ -328,7 +328,7 @@ func (c Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, bu for i := 1; i < partNumber; i++ { part, ok := partsInfo[i] if !ok { - return 0, ErrInvalidArgument(fmt.Sprintf("Missing part number %d", i)) + return 0, errInvalidArgument(fmt.Sprintf("Missing part number %d", i)) } complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ ETag: part.ETag, @@ -361,11 +361,11 @@ func (c Client) putObject(ctx context.Context, bucketName, objectName string, re // Size -1 is only supported on Google Cloud Storage, we error // out in all other situations. if size < 0 && !s3utils.IsGoogleEndpoint(*c.endpointURL) { - return 0, ErrEntityTooSmall(size, bucketName, objectName) + return 0, errEntityTooSmall(size, bucketName, objectName) } if opts.SendContentMd5 && s3utils.IsGoogleEndpoint(*c.endpointURL) && size < 0 { - return 0, ErrInvalidArgument("MD5Sum cannot be calculated with size '-1'") + return 0, errInvalidArgument("MD5Sum cannot be calculated with size '-1'") } if size > 0 { @@ -374,7 +374,7 @@ func (c Client) putObject(ctx context.Context, bucketName, objectName string, re if ok { offset, err := seeker.Seek(0, io.SeekCurrent) if err != nil { - return 0, ErrInvalidArgument(err.Error()) + return 0, errInvalidArgument(err.Error()) } reader = io.NewSectionReader(reader.(io.ReaderAt), offset, size) } @@ -410,7 +410,7 @@ func (c Client) putObject(ctx context.Context, bucketName, objectName string, re return 0, err } if st.Size != size { - return 0, ErrUnexpectedEOF(st.Size, size, bucketName, objectName) + return 0, errUnexpectedEOF(st.Size, size, bucketName, objectName) } return size, nil } diff --git a/api-put-object.go b/api-put-object.go index a9a2b4512..4c8dd2829 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -137,19 +137,19 @@ func (opts PutObjectOptions) Header() (header http.Header) { func (opts PutObjectOptions) validate() (err error) { for k, v := range opts.UserMetadata { if !httpguts.ValidHeaderFieldName(k) || isStandardHeader(k) || isSSEHeader(k) || isStorageClassHeader(k) { - return ErrInvalidArgument(k + " unsupported user defined metadata name") + return errInvalidArgument(k + " unsupported user defined metadata name") } if !httpguts.ValidHeaderFieldValue(v) { - return ErrInvalidArgument(v + " unsupported user defined metadata value") + return errInvalidArgument(v + " unsupported user defined metadata value") } } if opts.Mode != nil { if !opts.Mode.IsValid() { - return ErrInvalidArgument(opts.Mode.String() + " unsupported retention mode") + return errInvalidArgument(opts.Mode.String() + " unsupported retention mode") } } if opts.LegalHold != "" && !opts.LegalHold.IsValid() { - return ErrInvalidArgument(opts.LegalHold.String() + " unsupported legal-hold status") + return errInvalidArgument(opts.LegalHold.String() + " unsupported legal-hold status") } return nil } @@ -190,7 +190,7 @@ func (c Client) PutObject(ctx context.Context, bucketName, objectName string, re func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) { // Check for largest object size allowed. if size > int64(maxMultipartPutObjectSize) { - return 0, ErrEntityTooLarge(size, maxMultipartPutObjectSize, bucketName, objectName) + return 0, errEntityTooLarge(size, maxMultipartPutObjectSize, bucketName, objectName) } // NOTE: Streaming signature is not supported by GCS. @@ -312,7 +312,7 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName for i := 1; i < partNumber; i++ { part, ok := partsInfo[i] if !ok { - return 0, ErrInvalidArgument(fmt.Sprintf("Missing part number %d", i)) + return 0, errInvalidArgument(fmt.Sprintf("Missing part number %d", i)) } complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ ETag: part.ETag, diff --git a/api-remove.go b/api-remove.go index 415cefb03..df4d2f331 100644 --- a/api-remove.go +++ b/api-remove.go @@ -176,7 +176,7 @@ func (c Client) RemoveObjects(ctx context.Context, bucketName string, objectsCh if objectsCh == nil { defer close(errorCh) errorCh <- RemoveObjectError{ - Err: ErrInvalidArgument("Objects channel cannot be nil"), + Err: errInvalidArgument("Objects channel cannot be nil"), } return errorCh } @@ -281,7 +281,7 @@ func (c Client) RemoveObjectsWithOptions(ctx context.Context, bucketName string, if objectsCh == nil { defer close(errorCh) errorCh <- RemoveObjectError{ - Err: ErrInvalidArgument("Objects channel cannot be nil"), + Err: errInvalidArgument("Objects channel cannot be nil"), } return errorCh } diff --git a/api.go b/api.go index 98c928cda..d833ba7a6 100644 --- a/api.go +++ b/api.go @@ -567,7 +567,7 @@ func (c Client) do(req *http.Request) (*http.Response, error) { // Response cannot be non-nil, report error if thats the case. if resp == nil { msg := "Response is empty. " + reportIssue - return nil, ErrInvalidArgument(msg) + return nil, errInvalidArgument(msg) } // If trace is enabled, dump http request and response, @@ -822,7 +822,7 @@ func (c Client) newRequest(ctx context.Context, method string, metadata requestM // Generate presign url if needed, return right here. if metadata.expires != 0 && metadata.presignURL { if signerType.IsAnonymous() { - return nil, ErrInvalidArgument("Presigned URLs cannot be generated with anonymous credentials.") + return nil, errInvalidArgument("Presigned URLs cannot be generated with anonymous credentials.") } if signerType.IsV2() { // Presign URL with signature v2. @@ -912,7 +912,7 @@ func (c Client) makeTargetURL(bucketName, objectName, bucketLocation string, isV // http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html // Disable transfer acceleration for non-compliant bucket names. if strings.Contains(bucketName, ".") { - return nil, ErrTransferAccelerationBucket(bucketName) + return nil, errTransferAccelerationBucket(bucketName) } // If transfer acceleration is requested set new host. // For more details about enabling transfer acceleration read here. diff --git a/post-policy.go b/post-policy.go index 5b84486a4..98d23550d 100644 --- a/post-policy.go +++ b/post-policy.go @@ -73,7 +73,7 @@ func NewPostPolicy() *PostPolicy { // SetExpires - Sets expiration time for the new policy. func (p *PostPolicy) SetExpires(t time.Time) error { if t.IsZero() { - return ErrInvalidArgument("No expiry time set.") + return errInvalidArgument("No expiry time set.") } p.expiration = t return nil @@ -82,7 +82,7 @@ func (p *PostPolicy) SetExpires(t time.Time) error { // SetKey - Sets an object name for the policy based upload. func (p *PostPolicy) SetKey(key string) error { if strings.TrimSpace(key) == "" || key == "" { - return ErrInvalidArgument("Object name is empty.") + return errInvalidArgument("Object name is empty.") } policyCond := policyCondition{ matchType: "eq", @@ -100,7 +100,7 @@ func (p *PostPolicy) SetKey(key string) error { // can start with. func (p *PostPolicy) SetKeyStartsWith(keyStartsWith string) error { if strings.TrimSpace(keyStartsWith) == "" || keyStartsWith == "" { - return ErrInvalidArgument("Object prefix is empty.") + return errInvalidArgument("Object prefix is empty.") } policyCond := policyCondition{ matchType: "starts-with", @@ -117,7 +117,7 @@ func (p *PostPolicy) SetKeyStartsWith(keyStartsWith string) error { // SetBucket - Sets bucket at which objects will be uploaded to. func (p *PostPolicy) SetBucket(bucketName string) error { if strings.TrimSpace(bucketName) == "" || bucketName == "" { - return ErrInvalidArgument("Bucket name is empty.") + return errInvalidArgument("Bucket name is empty.") } policyCond := policyCondition{ matchType: "eq", @@ -134,7 +134,7 @@ func (p *PostPolicy) SetBucket(bucketName string) error { // SetCondition - Sets condition for credentials, date and algorithm func (p *PostPolicy) SetCondition(matchType, condition, value string) error { if strings.TrimSpace(value) == "" || value == "" { - return ErrInvalidArgument("No value specified for condition") + return errInvalidArgument("No value specified for condition") } policyCond := policyCondition{ @@ -149,14 +149,14 @@ func (p *PostPolicy) SetCondition(matchType, condition, value string) error { p.formData[condition] = value return nil } - return ErrInvalidArgument("Invalid condition in policy") + return errInvalidArgument("Invalid condition in policy") } // SetContentType - Sets content-type of the object for this policy // based upload. func (p *PostPolicy) SetContentType(contentType string) error { if strings.TrimSpace(contentType) == "" || contentType == "" { - return ErrInvalidArgument("No content type specified.") + return errInvalidArgument("No content type specified.") } policyCond := policyCondition{ matchType: "eq", @@ -174,13 +174,13 @@ func (p *PostPolicy) SetContentType(contentType string) error { // condition for all incoming uploads. func (p *PostPolicy) SetContentLengthRange(min, max int64) error { if min > max { - return ErrInvalidArgument("Minimum limit is larger than maximum limit.") + return errInvalidArgument("Minimum limit is larger than maximum limit.") } if min < 0 { - return ErrInvalidArgument("Minimum limit cannot be negative.") + return errInvalidArgument("Minimum limit cannot be negative.") } if max < 0 { - return ErrInvalidArgument("Maximum limit cannot be negative.") + return errInvalidArgument("Maximum limit cannot be negative.") } p.contentLengthRange.min = min p.contentLengthRange.max = max @@ -191,7 +191,7 @@ func (p *PostPolicy) SetContentLengthRange(min, max int64) error { // based upload. func (p *PostPolicy) SetSuccessStatusAction(status string) error { if strings.TrimSpace(status) == "" || status == "" { - return ErrInvalidArgument("Status is empty") + return errInvalidArgument("Status is empty") } policyCond := policyCondition{ matchType: "eq", @@ -209,10 +209,10 @@ func (p *PostPolicy) SetSuccessStatusAction(status string) error { // Can be retrieved through a HEAD request or an event. func (p *PostPolicy) SetUserMetadata(key string, value string) error { if strings.TrimSpace(key) == "" || key == "" { - return ErrInvalidArgument("Key is empty") + return errInvalidArgument("Key is empty") } if strings.TrimSpace(value) == "" || value == "" { - return ErrInvalidArgument("Value is empty") + return errInvalidArgument("Value is empty") } headerName := fmt.Sprintf("x-amz-meta-%s", key) policyCond := policyCondition{ @@ -231,10 +231,10 @@ func (p *PostPolicy) SetUserMetadata(key string, value string) error { // Can be retrieved through a HEAD request or an event. func (p *PostPolicy) SetUserData(key string, value string) error { if key == "" { - return ErrInvalidArgument("Key is empty") + return errInvalidArgument("Key is empty") } if value == "" { - return ErrInvalidArgument("Value is empty") + return errInvalidArgument("Value is empty") } headerName := fmt.Sprintf("x-amz-%s", key) policyCond := policyCondition{ @@ -252,7 +252,7 @@ func (p *PostPolicy) SetUserData(key string, value string) error { // addNewPolicy - internal helper to validate adding new policies. func (p *PostPolicy) addNewPolicy(policyCond policyCondition) error { if policyCond.matchType == "" || policyCond.condition == "" || policyCond.value == "" { - return ErrInvalidArgument("Policy fields are empty.") + return errInvalidArgument("Policy fields are empty.") } p.conditions = append(p.conditions, policyCond) return nil diff --git a/utils.go b/utils.go index ab20028b6..f5400d95a 100644 --- a/utils.go +++ b/utils.go @@ -75,12 +75,12 @@ func getEndpointURL(endpoint string, secure bool) (*url.URL, error) { } if !s3utils.IsValidIP(host) && !s3utils.IsValidDomain(host) { msg := "Endpoint: " + endpoint + " does not follow ip address or domain name standards." - return nil, ErrInvalidArgument(msg) + return nil, errInvalidArgument(msg) } } else { if !s3utils.IsValidIP(endpoint) && !s3utils.IsValidDomain(endpoint) { msg := "Endpoint: " + endpoint + " does not follow ip address or domain name standards." - return nil, ErrInvalidArgument(msg) + return nil, errInvalidArgument(msg) } } // If secure is false, use 'http' scheme. @@ -134,19 +134,19 @@ var ( // Verify if input endpoint URL is valid. func isValidEndpointURL(endpointURL url.URL) error { if endpointURL == sentinelURL { - return ErrInvalidArgument("Endpoint url cannot be empty.") + return errInvalidArgument("Endpoint url cannot be empty.") } if endpointURL.Path != "/" && endpointURL.Path != "" { - return ErrInvalidArgument("Endpoint url cannot have fully qualified paths.") + return errInvalidArgument("Endpoint url cannot have fully qualified paths.") } if strings.Contains(endpointURL.Host, ".s3.amazonaws.com") { if !s3utils.IsAmazonEndpoint(endpointURL) { - return ErrInvalidArgument("Amazon S3 endpoint should be 's3.amazonaws.com'.") + return errInvalidArgument("Amazon S3 endpoint should be 's3.amazonaws.com'.") } } if strings.Contains(endpointURL.Host, ".googleapis.com") { if !s3utils.IsGoogleEndpoint(endpointURL) { - return ErrInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'.") + return errInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'.") } } return nil @@ -156,10 +156,10 @@ func isValidEndpointURL(endpointURL url.URL) error { func isValidExpiry(expires time.Duration) error { expireSeconds := int64(expires / time.Second) if expireSeconds < 1 { - return ErrInvalidArgument("Expires cannot be lesser than 1 second.") + return errInvalidArgument("Expires cannot be lesser than 1 second.") } if expireSeconds > 604800 { - return ErrInvalidArgument("Expires cannot be greater than 7 days.") + return errInvalidArgument("Expires cannot be greater than 7 days.") } return nil } diff --git a/utils_test.go b/utils_test.go index 856d7319a..e8628d229 100644 --- a/utils_test.go +++ b/utils_test.go @@ -73,11 +73,11 @@ func TestGetEndpointURL(t *testing.T) { {"192.168.1.1:9000", false, "http://192.168.1.1:9000", nil, true}, {"192.168.1.1:9000", true, "https://192.168.1.1:9000", nil, true}, {"s3.amazonaws.com:443", true, "https://s3.amazonaws.com:443", nil, true}, - {"13333.123123.-", true, "", ErrInvalidArgument(fmt.Sprintf("Endpoint: %s does not follow ip address or domain name standards.", "13333.123123.-")), false}, - {"13333.123123.-", true, "", ErrInvalidArgument(fmt.Sprintf("Endpoint: %s does not follow ip address or domain name standards.", "13333.123123.-")), false}, - {"storage.googleapis.com:4000", true, "", ErrInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'."), false}, - {"s3.aamzza.-", true, "", ErrInvalidArgument(fmt.Sprintf("Endpoint: %s does not follow ip address or domain name standards.", "s3.aamzza.-")), false}, - {"", true, "", ErrInvalidArgument("Endpoint: does not follow ip address or domain name standards."), false}, + {"13333.123123.-", true, "", errInvalidArgument(fmt.Sprintf("Endpoint: %s does not follow ip address or domain name standards.", "13333.123123.-")), false}, + {"13333.123123.-", true, "", errInvalidArgument(fmt.Sprintf("Endpoint: %s does not follow ip address or domain name standards.", "13333.123123.-")), false}, + {"storage.googleapis.com:4000", true, "", errInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'."), false}, + {"s3.aamzza.-", true, "", errInvalidArgument(fmt.Sprintf("Endpoint: %s does not follow ip address or domain name standards.", "s3.aamzza.-")), false}, + {"", true, "", errInvalidArgument("Endpoint: does not follow ip address or domain name standards."), false}, } for i, testCase := range testCases { @@ -112,7 +112,7 @@ func TestIsValidEndpointURL(t *testing.T) { // Flag indicating whether the test is expected to pass or not. shouldPass bool }{ - {"", ErrInvalidArgument("Endpoint url cannot be empty."), false}, + {"", errInvalidArgument("Endpoint url cannot be empty."), false}, {"/", nil, true}, {"https://s3.amazonaws.com", nil, true}, {"https://s3.cn-north-1.amazonaws.com.cn", nil, true}, @@ -122,10 +122,10 @@ func TestIsValidEndpointURL(t *testing.T) { {"https://storage.googleapis.com/", nil, true}, {"https://z3.amazonaws.com", nil, true}, {"https://mybalancer.us-east-1.elb.amazonaws.com", nil, true}, - {"192.168.1.1", ErrInvalidArgument("Endpoint url cannot have fully qualified paths."), false}, - {"https://amazon.googleapis.com/", ErrInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'."), false}, - {"https://storage.googleapis.com/bucket/", ErrInvalidArgument("Endpoint url cannot have fully qualified paths."), false}, - {"https://s3.amazonaws.com/bucket/object", ErrInvalidArgument("Endpoint url cannot have fully qualified paths."), false}, + {"192.168.1.1", errInvalidArgument("Endpoint url cannot have fully qualified paths."), false}, + {"https://amazon.googleapis.com/", errInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'."), false}, + {"https://storage.googleapis.com/bucket/", errInvalidArgument("Endpoint url cannot have fully qualified paths."), false}, + {"https://s3.amazonaws.com/bucket/object", errInvalidArgument("Endpoint url cannot have fully qualified paths."), false}, } for i, testCase := range testCases { @@ -218,9 +218,9 @@ func TestIsValidExpiry(t *testing.T) { // Flag to indicate whether the test should pass. shouldPass bool }{ - {100 * time.Millisecond, ErrInvalidArgument("Expires cannot be lesser than 1 second."), false}, - {604801 * time.Second, ErrInvalidArgument("Expires cannot be greater than 7 days."), false}, - {0 * time.Second, ErrInvalidArgument("Expires cannot be lesser than 1 second."), false}, + {100 * time.Millisecond, errInvalidArgument("Expires cannot be lesser than 1 second."), false}, + {604801 * time.Second, errInvalidArgument("Expires cannot be greater than 7 days."), false}, + {0 * time.Second, errInvalidArgument("Expires cannot be lesser than 1 second."), false}, {1 * time.Second, nil, true}, {10000 * time.Second, nil, true}, {999 * time.Second, nil, true}, @@ -254,12 +254,12 @@ func TestIsValidBucketName(t *testing.T) { // Flag to indicate whether test should Pass. shouldPass bool }{ - {".mybucket", ErrInvalidBucketName("Bucket name contains invalid characters"), false}, - {"mybucket.", ErrInvalidBucketName("Bucket name contains invalid characters"), false}, - {"mybucket-", ErrInvalidBucketName("Bucket name contains invalid characters"), false}, - {"my", ErrInvalidBucketName("Bucket name cannot be shorter than 3 characters"), false}, - {"", ErrInvalidBucketName("Bucket name cannot be empty"), false}, - {"my..bucket", ErrInvalidBucketName("Bucket name contains invalid characters"), false}, + {".mybucket", errInvalidBucketName("Bucket name contains invalid characters"), false}, + {"mybucket.", errInvalidBucketName("Bucket name contains invalid characters"), false}, + {"mybucket-", errInvalidBucketName("Bucket name contains invalid characters"), false}, + {"my", errInvalidBucketName("Bucket name cannot be shorter than 3 characters"), false}, + {"", errInvalidBucketName("Bucket name cannot be empty"), false}, + {"my..bucket", errInvalidBucketName("Bucket name contains invalid characters"), false}, {"my.bucket.com", nil, true}, {"my-bucket", nil, true}, {"123my-bucket", nil, true}, From 47d67c2ef9400369aced77703ba79a27344ea873 Mon Sep 17 00:00:00 2001 From: Didier Franc Date: Wed, 1 Jul 2020 21:35:59 +0200 Subject: [PATCH 170/215] fix: add missing "success_action_redirect" policy condition (#1313) --- post-policy.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/post-policy.go b/post-policy.go index 98d23550d..fa6add2d9 100644 --- a/post-policy.go +++ b/post-policy.go @@ -187,6 +187,24 @@ func (p *PostPolicy) SetContentLengthRange(min, max int64) error { return nil } +// SetSuccessActionRedirect - Sets the redirect success url of the object for this policy +// based upload. +func (p *PostPolicy) SetSuccessActionRedirect(redirect string) error { + if strings.TrimSpace(redirect) == "" || redirect == "" { + return ErrInvalidArgument("Redirect is empty") + } + policyCond := policyCondition{ + matchType: "eq", + condition: "$success_action_redirect", + value: redirect, + } + if err := p.addNewPolicy(policyCond); err != nil { + return err + } + p.formData["success_action_redirect"] = redirect + return nil +} + // SetSuccessStatusAction - Sets the status success code of the object for this policy // based upload. func (p *PostPolicy) SetSuccessStatusAction(status string) error { From 144bc342fb2edf325e30b2bae210d7d6a4b521d5 Mon Sep 17 00:00:00 2001 From: He Liu Date: Wed, 1 Jul 2020 20:28:38 -0700 Subject: [PATCH 171/215] missing errInvalidArgument (#1323) --- api-error-response_test.go | 2 +- post-policy.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api-error-response_test.go b/api-error-response_test.go index 8e823183b..d7aa8c5ab 100644 --- a/api-error-response_test.go +++ b/api-error-response_test.go @@ -260,7 +260,7 @@ func TestErrInvalidObjectName(t *testing.T) { } } -// Test validates 'ErrInvalidArgument' response. +// Test validates 'errInvalidArgument' response. func TestErrInvalidArgument(t *testing.T) { expectedResult := ErrorResponse{ StatusCode: http.StatusBadRequest, diff --git a/post-policy.go b/post-policy.go index fa6add2d9..d489d981a 100644 --- a/post-policy.go +++ b/post-policy.go @@ -191,7 +191,7 @@ func (p *PostPolicy) SetContentLengthRange(min, max int64) error { // based upload. func (p *PostPolicy) SetSuccessActionRedirect(redirect string) error { if strings.TrimSpace(redirect) == "" || redirect == "" { - return ErrInvalidArgument("Redirect is empty") + return errInvalidArgument("Redirect is empty") } policyCond := policyCondition{ matchType: "eq", From a5c6267d8d9ac063e7b1550ae7e224b54e592aa1 Mon Sep 17 00:00:00 2001 From: Anis Elleuch Date: Fri, 3 Jul 2020 07:36:36 +0100 Subject: [PATCH 172/215] refactor: GetObjectTagging returns a map of key/value (#1318) --- api-object-tagging.go | 13 ++++++------- docs/API.md | 4 ++-- examples/s3/getobjecttagging.go | 4 +++- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/api-object-tagging.go b/api-object-tagging.go index 42b6f62a5..ec8a161cb 100644 --- a/api-object-tagging.go +++ b/api-object-tagging.go @@ -21,7 +21,6 @@ import ( "bytes" "context" "encoding/xml" - "io/ioutil" "net/http" "net/url" @@ -77,7 +76,7 @@ func (c Client) PutObjectTagging(ctx context.Context, bucketName, objectName str // GetObjectTagging fetches object tag(s) with a context to control cancellations // and timeouts. -func (c Client) GetObjectTagging(ctx context.Context, bucketName, objectName string) (string, error) { +func (c Client) GetObjectTagging(ctx context.Context, bucketName, objectName string) (map[string]string, error) { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) @@ -92,21 +91,21 @@ func (c Client) GetObjectTagging(ctx context.Context, bucketName, objectName str defer closeResponse(resp) if err != nil { - return "", err + return nil, err } if resp != nil { if resp.StatusCode != http.StatusOK { - return "", httpRespToErrorResponse(resp, bucketName, objectName) + return nil, httpRespToErrorResponse(resp, bucketName, objectName) } } - tagBuf, err := ioutil.ReadAll(resp.Body) + tags, err := tags.ParseObjectXML(resp.Body) if err != nil { - return "", err + return nil, err } - return string(tagBuf), err + return tags.ToMap(), err } // RemoveObjectTagging removes object tag(s) with a context to control cancellations diff --git a/docs/API.md b/docs/API.md index d8c180647..d29bf3a89 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1354,7 +1354,7 @@ if err != nil { ``` -### GetObjectTagging(ctx context.Context, bucketName, objectName string) (string, error) +### GetObjectTagging(ctx context.Context, bucketName, objectName string) (_map[string]string_, error) Fetch Object Tags from the given object __Parameters__ @@ -1375,7 +1375,7 @@ if err != nil { fmt.Println(err) return } -fmt.Printf("Fetched Tags: %s", tags) +fmt.Printf("Fetched Tags: %v", tags) ``` diff --git a/examples/s3/getobjecttagging.go b/examples/s3/getobjecttagging.go index ffd754e23..9d03a5305 100644 --- a/examples/s3/getobjecttagging.go +++ b/examples/s3/getobjecttagging.go @@ -40,9 +40,11 @@ func main() { if err != nil { log.Fatalln(err) } + tags, err := s3Client.GetObjectTagging(context.Background(), "my-bucketname", "my-objectname") if err != nil { log.Fatalln(err) } - fmt.Printf("Fetched Object Tags: %s", tags) + + fmt.Printf("Fetched Object Tags: %v", tags) } From 7418ee0041fcf10d37f51fc8bb1712a73743438e Mon Sep 17 00:00:00 2001 From: ebozduman Date: Fri, 3 Jul 2020 00:26:59 -0700 Subject: [PATCH 173/215] Ignore duplicate entries in UserMetadata (#1315) --- api-compose-object.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-compose-object.go b/api-compose-object.go index ec8c05577..a04d37d65 100644 --- a/api-compose-object.go +++ b/api-compose-object.go @@ -79,7 +79,7 @@ func filterCustomMeta(userMeta map[string]string) (map[string]string, error) { k = k[len("x-amz-meta-"):] } if _, ok := m[k]; ok { - return nil, errInvalidArgument(fmt.Sprintf("Cannot add both %s and x-amz-meta-%s keys as custom metadata", k, k)) + continue } m[k] = v } From 051da3cb189143687e00113555775b1473eac19f Mon Sep 17 00:00:00 2001 From: Anis Elleuch Date: Fri, 3 Jul 2020 18:14:46 +0100 Subject: [PATCH 174/215] refactor: Single MakeBucket API with options (#1327) Use the following struct as options when creating a bucket: MakeBucketOptions { Region string ObjectLocking bool } MakeBucket(ctx context.Context, bucketName string, opts MakeBucketOptions) error --- api-put-bucket.go | 29 ++++----- core_test.go | 14 ++-- functional_tests.go | 152 ++++++++++++++++++++++---------------------- 3 files changed, 96 insertions(+), 99 deletions(-) diff --git a/api-put-bucket.go b/api-put-bucket.go index 6db66c756..13178e7c6 100644 --- a/api-put-bucket.go +++ b/api-put-bucket.go @@ -50,16 +50,16 @@ type ServerSideEncryptionConfiguration struct { /// Bucket operations -func (c Client) makeBucket(ctx context.Context, bucketName string, location string, objectLockEnabled bool) (err error) { +func (c Client) makeBucket(ctx context.Context, bucketName string, opts MakeBucketOptions) (err error) { // Validate the input arguments. if err := s3utils.CheckValidBucketNameStrict(bucketName); err != nil { return err } - err = c.doMakeBucket(ctx, bucketName, location, objectLockEnabled) - if err != nil && (location == "" || location == "us-east-1") { + err = c.doMakeBucket(ctx, bucketName, opts.Region, opts.ObjectLocking) + if err != nil && (opts.Region == "" || opts.Region == "us-east-1") { if resp, ok := err.(ErrorResponse); ok && resp.Code == "AuthorizationHeaderMalformed" && resp.Region != "" { - err = c.doMakeBucket(ctx, bucketName, resp.Region, objectLockEnabled) + err = c.doMakeBucket(ctx, bucketName, resp.Region, opts.ObjectLocking) } } return err @@ -126,26 +126,23 @@ func (c Client) doMakeBucket(ctx context.Context, bucketName string, location st return nil } -// MakeBucket creates a new bucket with bucketName with a context to control cancellations and timeouts. -// -// Location is an optional argument, by default all buckets are -// created in US Standard Region. -// -// For Amazon S3 for more supported regions - http://docs.aws.amazon.com/general/latest/gr/rande.html -// For Google Cloud Storage for more supported regions - https://cloud.google.com/storage/docs/bucket-locations -func (c Client) MakeBucket(ctx context.Context, bucketName string, location string) (err error) { - return c.makeBucket(ctx, bucketName, location, false) +// MakeBucketOptions holds all options to tweak bucket creation +type MakeBucketOptions struct { + // Bucket location + Region string + // Enable object locking + ObjectLocking bool } -// MakeBucketWithObjectLock creates a object lock enabled new bucket with bucketName. +// MakeBucket creates a new bucket with bucketName with a context to control cancellations and timeouts. // // Location is an optional argument, by default all buckets are // created in US Standard Region. // // For Amazon S3 for more supported regions - http://docs.aws.amazon.com/general/latest/gr/rande.html // For Google Cloud Storage for more supported regions - https://cloud.google.com/storage/docs/bucket-locations -func (c Client) MakeBucketWithObjectLock(ctx context.Context, bucketName string, location string) (err error) { - return c.makeBucket(ctx, bucketName, location, true) +func (c Client) MakeBucket(ctx context.Context, bucketName string, opts MakeBucketOptions) (err error) { + return c.makeBucket(ctx, bucketName, opts) } // SetBucketPolicy sets the access permissions on an existing bucket. diff --git a/core_test.go b/core_test.go index 4cc52f669..5b0509e06 100644 --- a/core_test.go +++ b/core_test.go @@ -93,7 +93,7 @@ func TestGetObjectCore(t *testing.T) { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, MakeBucketOptions{Region: "us-east-1"}) if err != nil { t.Fatal("Error:", err, bucketName) } @@ -286,7 +286,7 @@ func TestGetObjectContentEncoding(t *testing.T) { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, MakeBucketOptions{Region: "us-east-1"}) if err != nil { t.Fatal("Error:", err, bucketName) } @@ -363,7 +363,7 @@ func TestGetBucketPolicy(t *testing.T) { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, MakeBucketOptions{Region: "us-east-1"}) if err != nil { t.Fatal("Error:", err, bucketName) } @@ -426,7 +426,7 @@ func TestCoreCopyObject(t *testing.T) { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, MakeBucketOptions{Region: "us-east-1"}) if err != nil { t.Fatal("Error:", err, bucketName) } @@ -544,7 +544,7 @@ func TestCoreCopyObjectPart(t *testing.T) { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, MakeBucketOptions{Region: "us-east-1"}) if err != nil { t.Fatal("Error:", err, bucketName) } @@ -693,7 +693,7 @@ func TestCorePutObject(t *testing.T) { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, MakeBucketOptions{Region: "us-east-1"}) if err != nil { t.Fatal("Error:", err, bucketName) } @@ -780,7 +780,7 @@ func TestCoreGetObjectMetadata(t *testing.T) { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = core.MakeBucket(context.Background(), bucketName, "us-east-1") + err = core.MakeBucket(context.Background(), bucketName, MakeBucketOptions{Region: "us-east-1"}) if err != nil { t.Fatal("Error:", err, bucketName) } diff --git a/functional_tests.go b/functional_tests.go index cf084ce3f..1984b8c11 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -322,11 +322,11 @@ func testMakeBucketError() { args["bucketName"] = bucketName // Make a new bucket in 'eu-central-1'. - if err = c.MakeBucket(context.Background(), bucketName, region); err != nil { + if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err != nil { logError(testName, function, args, startTime, "", "MakeBucket Failed", err) return } - if err = c.MakeBucket(context.Background(), bucketName, region); err == nil { + if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err == nil { logError(testName, function, args, startTime, "", "Bucket already exists", err) return } @@ -374,7 +374,7 @@ func testMetadataSizeLimit() { objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return @@ -458,7 +458,7 @@ func testMakeBucketRegions() { args["bucketName"] = bucketName // Make a new bucket in 'eu-central-1'. - if err = c.MakeBucket(context.Background(), bucketName, region); err != nil { + if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } @@ -474,7 +474,7 @@ func testMakeBucketRegions() { // virtual host style. region = "us-west-2" args["region"] = region - if err = c.MakeBucket(context.Background(), bucketName+".withperiod", region); err != nil { + if err = c.MakeBucket(context.Background(), bucketName+".withperiod", minio.MakeBucketOptions{Region: region}); err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } @@ -525,7 +525,7 @@ func testPutObjectReadAt() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return @@ -635,7 +635,7 @@ func testPutObjectWithMetadata() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return @@ -745,7 +745,7 @@ func testPutObjectWithContentLanguage() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -824,7 +824,7 @@ func testPutObjectStreaming() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -890,7 +890,7 @@ func testGetObjectSeekEnd() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -1022,7 +1022,7 @@ func testGetObjectClosedTwice() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -1117,7 +1117,7 @@ func testRemoveObjectsWithContext() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) } @@ -1214,7 +1214,7 @@ func testRemoveMultipleObjects() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -1301,7 +1301,7 @@ func testFPutObjectMultipart() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -1418,7 +1418,7 @@ func testFPutObject() { args["bucketName"] = bucketName args["location"] = location function = "MakeBucket(bucketName, location)" - err = c.MakeBucket(context.Background(), bucketName, location) + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: location}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -1597,7 +1597,7 @@ func testFPutObjectWithContext() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -1708,7 +1708,7 @@ func testFPutObjectWithContextV2() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -1818,7 +1818,7 @@ func testPutObjectWithContext() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket call failed", err) return @@ -1896,7 +1896,7 @@ func testGetObjectReadSeekFunctional() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -2071,7 +2071,7 @@ func testGetObjectReadAtFunctional() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -2256,7 +2256,7 @@ func testGetObjectReadAtWhenEOFWasReached() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -2384,7 +2384,7 @@ func testPresignedPostPolicy() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -2606,14 +2606,14 @@ func testCopyObject() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } // Make a new bucket in 'us-east-1' (destination bucket). - err = c.MakeBucket(context.Background(), bucketName+"-copy", "us-east-1") + err = c.MakeBucket(context.Background(), bucketName+"-copy", minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -2837,7 +2837,7 @@ func testSSECEncryptedGetObjectReadSeekFunctional() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -3025,7 +3025,7 @@ func testSSES3EncryptedGetObjectReadSeekFunctional() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -3211,7 +3211,7 @@ func testSSECEncryptedGetObjectReadAtFunctional() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -3402,7 +3402,7 @@ func testSSES3EncryptedGetObjectReadAtFunctional() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -3594,7 +3594,7 @@ func testSSECEncryptionPutGet() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -3708,7 +3708,7 @@ func testSSECEncryptionFPut() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -3835,7 +3835,7 @@ func testSSES3EncryptionPutGet() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -3947,7 +3947,7 @@ func testSSES3EncryptionFPut() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -4177,7 +4177,7 @@ func testFunctional() { function = "MakeBucket(bucketName, region)" functionAll = "MakeBucket(bucketName, region)" args["bucketName"] = bucketName - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) @@ -4811,7 +4811,7 @@ func testGetObjectModified() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -4911,7 +4911,7 @@ func testPutObjectUploadSeekedObject() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -5047,11 +5047,11 @@ func testMakeBucketErrorV2() { args["region"] = region // Make a new bucket in 'eu-west-1'. - if err = c.MakeBucket(context.Background(), bucketName, region); err != nil { + if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } - if err = c.MakeBucket(context.Background(), bucketName, region); err == nil { + if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err == nil { logError(testName, function, args, startTime, "", "MakeBucket did not fail for existing bucket name", err) return } @@ -5106,7 +5106,7 @@ func testGetObjectClosedTwiceV2() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -5206,7 +5206,7 @@ func testFPutObjectV2() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -5373,7 +5373,7 @@ func testMakeBucketRegionsV2() { args["bucketName"] = bucketName // Make a new bucket in 'eu-central-1'. - if err = c.MakeBucket(context.Background(), bucketName, "eu-west-1"); err != nil { + if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "eu-west-1"}); err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } @@ -5386,7 +5386,7 @@ func testMakeBucketRegionsV2() { // Make a new bucket with '.' in its name, in 'us-west-2'. This // request is internally staged into a path style instead of // virtual host style. - if err = c.MakeBucket(context.Background(), bucketName+".withperiod", "us-west-2"); err != nil { + if err = c.MakeBucket(context.Background(), bucketName+".withperiod", minio.MakeBucketOptions{Region: "us-west-2"}); err != nil { args["bucketName"] = bucketName + ".withperiod" args["region"] = "us-west-2" logError(testName, function, args, startTime, "", "MakeBucket failed", err) @@ -5436,7 +5436,7 @@ func testGetObjectReadSeekFunctionalV2() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -5600,7 +5600,7 @@ func testGetObjectReadAtFunctionalV2() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -5769,14 +5769,14 @@ func testCopyObjectV2() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } // Make a new bucket in 'us-east-1' (destination bucket). - err = c.MakeBucket(context.Background(), bucketName+"-copy", "us-east-1") + err = c.MakeBucket(context.Background(), bucketName+"-copy", minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -5941,7 +5941,7 @@ func testComposeObjectErrorCasesWrapper(c *minio.Client) { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). - err := c.MakeBucket(context.Background(), bucketName, "us-east-1") + err := c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) @@ -6042,7 +6042,7 @@ func testComposeMultipleSources(c *minio.Client) { // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). - err := c.MakeBucket(context.Background(), bucketName, "us-east-1") + err := c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -6147,7 +6147,7 @@ func testEncryptedEmptyObject() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -6233,7 +6233,7 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, var srcEncryption, dstEncryption encrypt.ServerSide // Make a new bucket in 'us-east-1' (source bucket). - err := c.MakeBucket(context.Background(), bucketName, "us-east-1") + err := c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -6674,7 +6674,7 @@ func testDecryptedCopyObject() { } bucketName, objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-"), "object" - if err = c.MakeBucket(context.Background(), bucketName, "us-east-1"); err != nil { + if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}); err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } @@ -6740,7 +6740,7 @@ func testSSECMultipartEncryptedToSSECCopyObjectPart() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) } @@ -6912,7 +6912,7 @@ func testSSECEncryptedToSSECCopyObjectPart() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) } @@ -7067,7 +7067,7 @@ func testSSECEncryptedToUnencryptedCopyPart() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) } @@ -7221,7 +7221,7 @@ func testSSECEncryptedToSSES3CopyObjectPart() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) } @@ -7378,7 +7378,7 @@ func testUnencryptedToSSECCopyObjectPart() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) } @@ -7530,7 +7530,7 @@ func testUnencryptedToUnencryptedCopyPart() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) } @@ -7679,7 +7679,7 @@ func testUnencryptedToSSES3CopyObjectPart() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) } @@ -7830,7 +7830,7 @@ func testSSES3EncryptedToSSECCopyObjectPart() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) } @@ -7983,7 +7983,7 @@ func testSSES3EncryptedToUnencryptedCopyPart() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) } @@ -8133,7 +8133,7 @@ func testSSES3EncryptedToSSES3CopyObjectPart() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) } @@ -8286,7 +8286,7 @@ func testUserMetadataCopyingWrapper(c *minio.Client) { // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). - err := c.MakeBucket(context.Background(), bucketName, "us-east-1") + err := c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -8485,7 +8485,7 @@ func testStorageClassMetadataPutObject() { // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -8575,7 +8575,7 @@ func testStorageClassInvalidMetadataPutObject() { // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -8620,7 +8620,7 @@ func testStorageClassMetadataCopyObject() { // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -8746,7 +8746,7 @@ func testPutObjectNoLengthV2() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -8820,7 +8820,7 @@ func testPutObjectsUnknownV2() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -8904,7 +8904,7 @@ func testPutObject0ByteV2() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -9020,7 +9020,7 @@ func testFunctionalV2() { "bucketName": bucketName, "location": location, } - err = c.MakeBucket(context.Background(), bucketName, location) + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: location}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -9453,7 +9453,7 @@ func testGetObjectWithContext() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -9561,7 +9561,7 @@ func testFGetObjectWithContext() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -9658,7 +9658,7 @@ func testGetObjectACLWithContext() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -9820,7 +9820,7 @@ func testPutObjectWithContextV2() { bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -9903,7 +9903,7 @@ func testGetObjectWithContextV2() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -10009,7 +10009,7 @@ func testFGetObjectWithContextV2() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket call failed", err) return @@ -10102,7 +10102,7 @@ func testListObjects() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return @@ -10211,7 +10211,7 @@ func testRemoveObjectsWithOptions() { args["objectName"] = objectName // Make a new bucket. - err = c.MakeBucketWithObjectLock(context.Background(), bucketName, "us-east-1") + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return From 764e9b8cecfcfd9968589d0a7b69a3b01a6afbcb Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 6 Jul 2020 06:41:25 -0700 Subject: [PATCH 175/215] upgrade golang-lint and fix the reported issues (#1329) --- Makefile | 13 +++-- NOTICE | 11 +++- code_of_conduct.md | 80 +++++++++++++++++++++++++++ go.mod | 7 +-- go.sum | 20 ++----- pkg/policy/bucket-policy-condition.go | 8 +-- 6 files changed, 107 insertions(+), 32 deletions(-) create mode 100644 code_of_conduct.md diff --git a/Makefile b/Makefile index afe8d978f..d9b433443 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,17 @@ +GOPATH := $(shell go env GOPATH) + all: checks .PHONY: examples docs -checks: diff vet test examples functional-test +checks: lint vet test examples functional-test -diff: - @curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b . v1.21.0 - @./golangci-lint run --timeout=5m --config ./.golangci.yml +lint: + @mkdir -p ${GOPATH}/bin + @which golangci-lint 1>/dev/null || (echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.27.0) + @echo "Running $@ check" + @GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean + @GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=5m --config ./.golangci.yml vet: @GO111MODULE=on go vet ./... diff --git a/NOTICE b/NOTICE index fc6c8808e..1e8fd3b92 100644 --- a/NOTICE +++ b/NOTICE @@ -1,2 +1,9 @@ -minio-go -Copyright 2015-2017 MinIO, Inc. \ No newline at end of file +MinIO Cloud Storage, (C) 2014-2020 MinIO, Inc. + +This product includes software developed at MinIO, Inc. +(https://min.io/). + +The MinIO project contains unmodified/modified subcomponents too with +separate copyright notices and license terms. Your use of the source +code for these subcomponents is subject to the terms and conditions +of Apache License Version 2.0 diff --git a/code_of_conduct.md b/code_of_conduct.md new file mode 100644 index 000000000..cb232c3c6 --- /dev/null +++ b/code_of_conduct.md @@ -0,0 +1,80 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior, in compliance with the +licensing terms applying to the Project developments. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. However, these actions shall respect the +licensing terms of the Project Developments that will always supersede such +Code of Conduct. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at dev@min.io. The project team +will review and investigate all complaints, and will respond in a way that it deems +appropriate to the circumstances. The project team is obligated to maintain +confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at [http://contributor-covenant.org/version/1/4][version] + +This version includes a clarification to ensure that the code of conduct is in +compliance with the free software licensing terms of the project. + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/4/ diff --git a/go.mod b/go.mod index 647c8c2d7..bdfb04133 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,13 @@ module github.com/minio/minio-go/v7 go 1.12 require ( - github.com/json-iterator/go v1.1.9 + github.com/json-iterator/go v1.1.10 github.com/minio/md5-simd v1.1.0 - github.com/minio/minio-go/v6 v6.0.57 // indirect github.com/minio/sha256-simd v0.1.1 github.com/mitchellh/go-homedir v1.1.0 - github.com/sirupsen/logrus v1.6.0 // indirect + github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f golang.org/x/net v0.0.0-20190522155817-f3200d17e092 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae // indirect - gopkg.in/ini.v1 v1.42.0 + gopkg.in/ini.v1 v1.57.0 ) diff --git a/go.sum b/go.sum index 3a1c62a77..6452387c3 100644 --- a/go.sum +++ b/go.sum @@ -1,24 +1,17 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= -github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= -github.com/minio/minio-go/v6 v6.0.57 h1:ixPkbKkyD7IhnluRgQpGSpHdpvNVaW6OD5R9IAO/9Tw= -github.com/minio/minio-go/v6 v6.0.57/go.mod h1:5+R/nM9Pwrh0vqF+HbYYDQ84wdUFPyXHkrdT4AIkifM= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -29,15 +22,11 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLD github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= -github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -50,11 +39,10 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= -gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/pkg/policy/bucket-policy-condition.go b/pkg/policy/bucket-policy-condition.go index ae10b589b..04da742dc 100644 --- a/pkg/policy/bucket-policy-condition.go +++ b/pkg/policy/bucket-policy-condition.go @@ -46,9 +46,7 @@ func (ckm ConditionKeyMap) Remove(key string, value set.StringSet) { // RemoveKey - removes key and its value. func (ckm ConditionKeyMap) RemoveKey(key string) { - if _, ok := ckm[key]; ok { - delete(ckm, key) - } + delete(ckm, key) } // CopyConditionKeyMap - returns new copy of given ConditionKeyMap. @@ -91,9 +89,7 @@ func (cond ConditionMap) Add(condKey string, condKeyMap ConditionKeyMap) { // Remove - removes condition key and its value. func (cond ConditionMap) Remove(condKey string) { - if _, ok := cond[condKey]; ok { - delete(cond, condKey) - } + delete(cond, condKey) } // mergeConditionMap - returns new ConditionMap which contains merged key/value of two ConditionMap. From e39de3077eb5f77c6fcbfe76f19add5d231788e5 Mon Sep 17 00:00:00 2001 From: Anis Elleuch Date: Mon, 6 Jul 2020 16:13:03 +0100 Subject: [PATCH 176/215] Remove doneCh from ListenBucketNotification and ListIncompleteUploads (#1331) --- api-list.go | 29 ++++++++-------------- api-notification.go | 16 ++++++------ docs/API.md | 22 +++------------- examples/minio/listenbucketnotification.go | 8 +----- examples/s3/listincompleteuploads.go | 8 +----- functional_tests.go | 6 ++--- 6 files changed, 27 insertions(+), 62 deletions(-) diff --git a/api-list.go b/api-list.go index 8aea481eb..709d3561a 100644 --- a/api-list.go +++ b/api-list.go @@ -516,28 +516,22 @@ func (c Client) listObjectsQuery(ctx context.Context, bucketName, objectPrefix, // objectPrefix from the specified bucket. If recursion is enabled // it would list all subdirectories and all its contents. // -// Your input parameters are just bucketName, objectPrefix, recursive -// and a done channel to pro-actively close the internal go routine. +// Your input parameters are just bucketName, objectPrefix, recursive. // If you enable recursive as 'true' this function will return back all // the multipart objects in a given bucket name. // // api := client.New(....) -// // Create a done channel. -// doneCh := make(chan struct{}) -// defer close(doneCh) // // Recurively list all objects in 'mytestbucket' // recursive := true -// for message := range api.ListIncompleteUploads("mytestbucket", "starthere", recursive) { +// for message := range api.ListIncompleteUploads(context.Background(), "mytestbucket", "starthere", recursive) { // fmt.Println(message) // } - -// ListIncompleteUploads is a wrapper for ListIncompleteUploadsWithContent -func (c Client) ListIncompleteUploads(ctx context.Context, bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo { - return c.listIncompleteUploads(ctx, bucketName, objectPrefix, recursive, doneCh) +func (c Client) ListIncompleteUploads(ctx context.Context, bucketName, objectPrefix string, recursive bool) <-chan ObjectMultipartInfo { + return c.listIncompleteUploads(ctx, bucketName, objectPrefix, recursive) } // listIncompleteUploads lists all incomplete uploads. -func (c Client) listIncompleteUploads(ctx context.Context, bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo { +func (c Client) listIncompleteUploads(ctx context.Context, bucketName, objectPrefix string, recursive bool) <-chan ObjectMultipartInfo { // Allocate channel for multipart uploads. objectMultipartStatCh := make(chan ObjectMultipartInfo, 1) // Delimiter is set to "/" by default. @@ -585,8 +579,8 @@ func (c Client) listIncompleteUploads(ctx context.Context, bucketName, objectPre select { // Send individual uploads here. case objectMultipartStatCh <- obj: - // If done channel return here. - case <-doneCh: + // If the context is canceled + case <-ctx.Done(): return } } @@ -596,8 +590,8 @@ func (c Client) listIncompleteUploads(ctx context.Context, bucketName, objectPre select { // Send delimited prefixes here. case objectMultipartStatCh <- ObjectMultipartInfo{Key: obj.Prefix, Size: 0}: - // If done channel return here. - case <-doneCh: + // If context is canceled. + case <-ctx.Done(): return } } @@ -735,11 +729,8 @@ func (c Client) findUploadIDs(ctx context.Context, bucketName, objectName string var uploadIDs []string // Make list incomplete uploads recursive. isRecursive := true - // Create done channel to cleanup the routine. - doneCh := make(chan struct{}) - defer close(doneCh) // List all incomplete uploads. - for mpUpload := range c.listIncompleteUploads(ctx, bucketName, objectName, isRecursive, doneCh) { + for mpUpload := range c.listIncompleteUploads(ctx, bucketName, objectName, isRecursive) { if mpUpload.Err != nil { return nil, mpUpload.Err } diff --git a/api-notification.go b/api-notification.go index 9d3b16452..81d6efe88 100644 --- a/api-notification.go +++ b/api-notification.go @@ -136,7 +136,7 @@ type NotificationInfo struct { } // ListenBucketNotification listen for bucket events, this is a MinIO specific API -func (c Client) ListenBucketNotification(ctx context.Context, bucketName, prefix, suffix string, events []string, doneCh <-chan struct{}) <-chan NotificationInfo { +func (c Client) ListenBucketNotification(ctx context.Context, bucketName, prefix, suffix string, events []string) <-chan NotificationInfo { notificationInfoCh := make(chan NotificationInfo, 1) const notificationCapacity = 1024 * 1024 notificationEventBuffer := make([]byte, notificationCapacity) @@ -150,7 +150,7 @@ func (c Client) ListenBucketNotification(ctx context.Context, bucketName, prefix case notificationInfoCh <- NotificationInfo{ Err: err, }: - case <-doneCh: + case <-ctx.Done(): } return } @@ -161,7 +161,7 @@ func (c Client) ListenBucketNotification(ctx context.Context, bucketName, prefix case notificationInfoCh <- NotificationInfo{ Err: errAPINotSupported("Listening for bucket notification is specific only to `minio` server endpoints"), }: - case <-doneCh: + case <-ctx.Done(): } return } @@ -192,7 +192,7 @@ func (c Client) ListenBucketNotification(ctx context.Context, bucketName, prefix case notificationInfoCh <- NotificationInfo{ Err: err, }: - case <-doneCh: + case <-ctx.Done(): } return } @@ -204,7 +204,7 @@ func (c Client) ListenBucketNotification(ctx context.Context, bucketName, prefix case notificationInfoCh <- NotificationInfo{ Err: errResponse, }: - case <-doneCh: + case <-ctx.Done(): } return } @@ -227,7 +227,7 @@ func (c Client) ListenBucketNotification(ctx context.Context, bucketName, prefix case notificationInfoCh <- NotificationInfo{ Err: err, }: - case <-doneCh: + case <-ctx.Done(): return } closeResponse(resp) @@ -236,7 +236,7 @@ func (c Client) ListenBucketNotification(ctx context.Context, bucketName, prefix // Send notificationInfo select { case notificationInfoCh <- notificationInfo: - case <-doneCh: + case <-ctx.Done(): closeResponse(resp) return } @@ -247,7 +247,7 @@ func (c Client) ListenBucketNotification(ctx context.Context, bucketName, prefix case notificationInfoCh <- NotificationInfo{ Err: err, }: - case <-doneCh: + case <-ctx.Done(): return } } diff --git a/docs/API.md b/docs/API.md index d29bf3a89..493205404 100644 --- a/docs/API.md +++ b/docs/API.md @@ -382,7 +382,7 @@ for object := range objectCh { ``` -### ListIncompleteUploads(ctx context.Context, bucketName, prefix string, recursive bool, doneCh chan struct{}) <- chan ObjectMultipartInfo +### ListIncompleteUploads(ctx context.Context, bucketName, prefix string, recursive bool) <- chan ObjectMultipartInfo Lists partially uploaded objects in a bucket. @@ -395,7 +395,6 @@ __Parameters__ |`bucketName` | _string_ |Name of the bucket | | `prefix` |_string_ | Prefix of objects that are partially uploaded | | `recursive` | _bool_ |`true` indicates recursive style listing and `false` indicates directory style listing delimited by '/'. | -|`doneCh` | _chan struct{}_ | A message on this channel ends the ListenIncompleteUploads iterator. | __Return Value__ @@ -416,14 +415,8 @@ __Example__ ```go -// Create a done channel to control 'ListObjects' go routine. -doneCh := make(chan struct{}) - -// Indicate to our routine to exit cleanly upon return. -defer close(doneCh) - isRecursive := true // Recursively list everything at 'myprefix' -multiPartObjectCh := minioClient.ListIncompleteUploads(context.Background(), "mybucket", "myprefix", isRecursive, doneCh) +multiPartObjectCh := minioClient.ListIncompleteUploads(context.Background(), "mybucket", "myprefix", isRecursive) for multiPartObject := range multiPartObjectCh { if multiPartObject.Err != nil { fmt.Println(multiPartObject.Err) @@ -1732,7 +1725,7 @@ if err != nil { ``` -### ListenBucketNotification(context context.Context, bucketName, prefix, suffix string, events []string, doneCh <-chan struct{}) <-chan NotificationInfo +### ListenBucketNotification(context context.Context, bucketName, prefix, suffix string, events []string) <-chan NotificationInfo ListenBucketNotification API receives bucket notification events through the notification channel. The returned notification channel has two fields 'Records' and 'Err'. - 'Records' holds the notifications received from the server. @@ -1749,7 +1742,6 @@ __Parameters__ |`prefix` | _string_ | Object key prefix to filter notifications for | |`suffix` | _string_ | Object key suffix to filter notifications for | |`events` | _[]string_ | Enables notifications for specific event types | -|`doneCh` | _chan struct{}_ | A message on this channel ends the ListenBucketNotification iterator | __Return Values__ @@ -1768,18 +1760,12 @@ __Example__ ```go -// Create a done channel to control 'ListenBucketNotification' go routine. -doneCh := make(chan struct{}) - -// Indicate a background go-routine to exit cleanly upon return. -defer close(doneCh) - // Listen for bucket notifications on "mybucket" filtered by prefix, suffix and events. for notificationInfo := range minioClient.ListenBucketNotification(context.Background(), "mybucket", "myprefix/", ".mysuffix", []string{ "s3:ObjectCreated:*", "s3:ObjectAccessed:*", "s3:ObjectRemoved:*", - }, doneCh) { + }) { if notificationInfo.Err != nil { fmt.Println(notificationInfo.Err) } diff --git a/examples/minio/listenbucketnotification.go b/examples/minio/listenbucketnotification.go index d5aab5d40..77eca42db 100644 --- a/examples/minio/listenbucketnotification.go +++ b/examples/minio/listenbucketnotification.go @@ -42,18 +42,12 @@ func main() { // s3Client.TraceOn(os.Stderr) - // Create a done channel to control 'ListenBucketNotification' go routine. - doneCh := make(chan struct{}) - - // Indicate to our routine to exit cleanly upon return. - defer close(doneCh) - // Listen for bucket notifications on "mybucket" filtered by prefix, suffix and events. for notificationInfo := range minioClient.ListenBucketNotification(context.Background(), "YOUR-BUCKET", "PREFIX", "SUFFIX", []string{ "s3:ObjectCreated:*", "s3:ObjectAccessed:*", "s3:ObjectRemoved:*", - }, doneCh) { + }) { if notificationInfo.Err != nil { log.Fatalln(notificationInfo.Err) } diff --git a/examples/s3/listincompleteuploads.go b/examples/s3/listincompleteuploads.go index 462dfefa2..8dfa192ab 100644 --- a/examples/s3/listincompleteuploads.go +++ b/examples/s3/listincompleteuploads.go @@ -41,14 +41,8 @@ func main() { log.Fatalln(err) } - // Create a done channel to control 'ListObjects' go routine. - doneCh := make(chan struct{}) - - // Indicate to our routine to exit cleanly upon return. - defer close(doneCh) - // List all multipart uploads from a bucket-name with a matching prefix. - for multipartObject := range s3Client.ListIncompleteUploads(context.Background(), "my-bucketname", "my-prefixname", true, doneCh) { + for multipartObject := range s3Client.ListIncompleteUploads(context.Background(), "my-bucketname", "my-prefixname", true) { if multipartObject.Err != nil { fmt.Println(multipartObject.Err) return diff --git a/functional_tests.go b/functional_tests.go index 1984b8c11..27ab851c4 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -159,7 +159,7 @@ func cleanupBucket(bucketName string, c *minio.Client) error { } } } - for objPartInfo := range c.ListIncompleteUploads(context.Background(), bucketName, "", true, doneCh) { + for objPartInfo := range c.ListIncompleteUploads(context.Background(), bucketName, "", true) { if objPartInfo.Err != nil { return objPartInfo.Err } @@ -4445,7 +4445,7 @@ func testFunctional() { "isRecursive": isRecursive, } - for objIncompl := range c.ListIncompleteUploads(context.Background(), bucketName, objectName, isRecursive, doneCh) { + for objIncompl := range c.ListIncompleteUploads(context.Background(), bucketName, objectName, isRecursive) { if objIncompl.Key != "" { incompObjNotFound = false break @@ -9170,7 +9170,7 @@ func testFunctionalV2() { "objectName": objectName, "isRecursive": isRecursive, } - for objIncompl := range c.ListIncompleteUploads(context.Background(), bucketName, objectName, isRecursive, doneCh) { + for objIncompl := range c.ListIncompleteUploads(context.Background(), bucketName, objectName, isRecursive) { if objIncompl.Key != "" { incompObjNotFound = false break From c752eca3ae9682be0ee23842d99367cd37e88b97 Mon Sep 17 00:00:00 2001 From: Anis Elleuch Date: Thu, 4 Jun 2020 18:16:05 +0100 Subject: [PATCH 177/215] Implement Versioning support --- api-compose-object.go | 84 +- api-compose-object_test.go | 2 +- api-datatypes.go | 26 +- api-get-object.go | 7 + api-get-options.go | 3 +- api-list.go | 355 +++- api-object-tagging.go | 47 +- api-put-object-copy.go | 16 +- api-put-object-file-context.go | 10 +- api-put-object-multipart.go | 61 +- api-put-object-streaming.go | 106 +- api-put-object.go | 36 +- api-remove.go | 75 +- api-s3-datatypes.go | 102 +- api-stat.go | 9 +- core.go | 2 +- core_test.go | 82 +- docs/API.md | 161 +- examples/s3/composeobject.go | 6 +- examples/s3/copyobject-with-new-tags.go | 5 +- examples/s3/copyobject.go | 4 +- examples/s3/fputencrypted-object.go | 4 +- ...objectoptions.go => listobjectversions.go} | 35 +- examples/s3/removeobject.go | 8 +- functional_tests.go | 1686 +++++++++++++---- go.sum | 11 +- 26 files changed, 2118 insertions(+), 825 deletions(-) rename examples/s3/{removeobjectoptions.go => listobjectversions.go} (57%) diff --git a/api-compose-object.go b/api-compose-object.go index a04d37d65..66582eb12 100644 --- a/api-compose-object.go +++ b/api-compose-object.go @@ -88,45 +88,7 @@ func filterCustomMeta(userMeta map[string]string) (map[string]string, error) { // NewDestinationInfo - creates a compose-object/copy-source // destination info object. -// -// `sse` is the key info for server-side-encryption with customer -// provided key. If it is nil, no encryption is performed. -// -// `userMeta` is the user-metadata key-value pairs to be set on the -// destination. The keys are automatically prefixed with `x-amz-meta-` -// if needed. If nil is passed, and if only a single source (of any -// size) is provided in the ComposeObject call, then metadata from the -// source is copied to the destination. -func NewDestinationInfo(bucket, object string, sse encrypt.ServerSide, userMeta map[string]string) (d DestinationInfo, err error) { - // Input validation. - if err = s3utils.CheckValidBucketName(bucket); err != nil { - return d, err - } - if err = s3utils.CheckValidObjectName(object); err != nil { - return d, err - } - m, err := filterCustomMeta(userMeta) - if err != nil { - return d, err - } - opts := DestInfoOptions{ - Encryption: sse, - UserMeta: m, - UserTags: nil, - ReplaceTags: false, - LegalHold: LegalHoldStatus(""), - Mode: RetentionMode(""), - } - return DestinationInfo{ - bucket: bucket, - object: object, - opts: opts, - }, nil -} - -// NewDestinationInfoWithOptions - creates a compose-object/copy-source -// destination info object. -func NewDestinationInfoWithOptions(bucket, object string, destOpts DestInfoOptions) (d DestinationInfo, err error) { +func NewDestinationInfo(bucket, object string, destOpts DestInfoOptions) (d DestinationInfo, err error) { // Input validation. if err = s3utils.CheckValidBucketName(bucket); err != nil { return d, err @@ -173,6 +135,7 @@ func (d *DestinationInfo) getUserMetaHeadersMap(withCopyDirectiveHeader bool) ma // server-side copying APIs. type SourceInfo struct { bucket, object string + versionID string start, end int64 encryption encrypt.ServerSide // Headers to send with the upload-part-copy request involving @@ -200,6 +163,16 @@ func NewSourceInfo(bucket, object string, sse encrypt.ServerSide) SourceInfo { return r } +// SetVersionID - Set the version ID of the source object +func (s *SourceInfo) SetVersionID(versionID string) error { + if versionID == "" { + return errInvalidArgument("version ID must be non-empty.") + } + s.versionID = versionID + s.Headers.Set("x-amz-copy-source", s3utils.EncodePath(s.bucket+"/"+s.object)+"?versionId="+versionID) + return nil +} + // SetRange - Set the start and end offset of the source object to be // copied. If this method is not called, the whole source object is // copied. @@ -257,6 +230,9 @@ func (s *SourceInfo) getProps(c Client) (size int64, etag string, userMeta map[s // headers are added to the stat request if given. var objInfo ObjectInfo opts := StatObjectOptions{GetObjectOptions{ServerSideEncryption: encrypt.SSE(s.encryption)}} + if s.versionID != "" { + opts.VersionID = s.versionID + } objInfo, err = c.statObject(context.Background(), s.bucket, s.object, opts) if err != nil { err = errInvalidArgument(fmt.Sprintf("Could not stat object - %s/%s: %v", s.bucket, s.object, err)) @@ -413,9 +389,9 @@ func (c Client) uploadPartCopy(ctx context.Context, bucket, object, uploadID str // and concatenates them into a new object using only server-side copying // operations. Optionally takes progress reader hook for applications to // look at current progress. -func (c Client) ComposeObjectWithProgress(ctx context.Context, dst DestinationInfo, srcs []SourceInfo, progress io.Reader) error { +func (c Client) ComposeObjectWithProgress(ctx context.Context, dst DestinationInfo, srcs []SourceInfo, progress io.Reader) (UploadInfo, error) { if len(srcs) < 1 || len(srcs) > maxPartsCount { - return errInvalidArgument("There must be as least one and up to 10000 source objects.") + return UploadInfo{}, errInvalidArgument("There must be as least one and up to 10000 source objects.") } srcSizes := make([]int64, len(srcs)) var totalSize, size, totalParts int64 @@ -425,13 +401,13 @@ func (c Client) ComposeObjectWithProgress(ctx context.Context, dst DestinationIn for i, src := range srcs { size, etags[i], srcUserMeta, err = src.getProps(c) if err != nil { - return err + return UploadInfo{}, err } // Error out if client side encryption is used in this source object when // more than one source objects are given. if len(srcs) > 1 && src.Headers.Get("x-amz-meta-x-amz-key") != "" { - return errInvalidArgument( + return UploadInfo{}, errInvalidArgument( fmt.Sprintf("Client side encryption is used in source object %s/%s", src.bucket, src.object)) } @@ -442,7 +418,7 @@ func (c Client) ComposeObjectWithProgress(ctx context.Context, dst DestinationIn // 0 <= src.start <= src.end // so only invalid case to check is: if src.end >= size { - return errInvalidArgument( + return UploadInfo{}, errInvalidArgument( fmt.Sprintf("SourceInfo %d has invalid segment-to-copy [%d, %d] (size is %d)", i, src.start, src.end, size)) } @@ -451,14 +427,14 @@ func (c Client) ComposeObjectWithProgress(ctx context.Context, dst DestinationIn // Only the last source may be less than `absMinPartSize` if size < absMinPartSize && i < len(srcs)-1 { - return errInvalidArgument( + return UploadInfo{}, errInvalidArgument( fmt.Sprintf("SourceInfo %d is too small (%d) and it is not the last part", i, size)) } // Is data to copy too large? totalSize += size if totalSize > maxMultipartPutObjectSize { - return errInvalidArgument(fmt.Sprintf("Cannot compose an object of size %d (> 5TiB)", totalSize)) + return UploadInfo{}, errInvalidArgument(fmt.Sprintf("Cannot compose an object of size %d (> 5TiB)", totalSize)) } // record source size @@ -468,7 +444,7 @@ func (c Client) ComposeObjectWithProgress(ctx context.Context, dst DestinationIn totalParts += partsRequired(size) // Do we need more parts than we are allowed? if totalParts > maxPartsCount { - return errInvalidArgument(fmt.Sprintf( + return UploadInfo{}, errInvalidArgument(fmt.Sprintf( "Your proposed compose object requires more than %d parts", maxPartsCount)) } } @@ -507,7 +483,7 @@ func (c Client) ComposeObjectWithProgress(ctx context.Context, dst DestinationIn uploadID, err := c.newUploadID(ctx, dst.bucket, dst.object, PutObjectOptions{ServerSideEncryption: dst.opts.Encryption, UserMetadata: metaHeaders}) if err != nil { - return err + return UploadInfo{}, err } // 3. Perform copy part uploads @@ -538,7 +514,7 @@ func (c Client) ComposeObjectWithProgress(ctx context.Context, dst DestinationIn complPart, err := c.uploadPartCopy(ctx, dst.bucket, dst.object, uploadID, partIndex, h) if err != nil { - return err + return UploadInfo{}, err } if progress != nil { io.CopyN(ioutil.Discard, progress, end-start+1) @@ -549,19 +525,21 @@ func (c Client) ComposeObjectWithProgress(ctx context.Context, dst DestinationIn } // 4. Make final complete-multipart request. - _, err = c.completeMultipartUpload(ctx, dst.bucket, dst.object, uploadID, + uploadInfo, err := c.completeMultipartUpload(ctx, dst.bucket, dst.object, uploadID, completeMultipartUpload{Parts: objParts}) if err != nil { - return err + return UploadInfo{}, err } - return nil + + uploadInfo.Size = totalSize + return uploadInfo, nil } // ComposeObject - creates an object using server-side copying of // existing objects. It takes a list of source objects (with optional // offsets) and concatenates them into a new object using only // server-side copying operations. -func (c Client) ComposeObject(ctx context.Context, dst DestinationInfo, srcs []SourceInfo) error { +func (c Client) ComposeObject(ctx context.Context, dst DestinationInfo, srcs []SourceInfo) (UploadInfo, error) { return c.ComposeObjectWithProgress(ctx, dst, srcs, nil) } diff --git a/api-compose-object_test.go b/api-compose-object_test.go index ddb26a114..ab54af077 100644 --- a/api-compose-object_test.go +++ b/api-compose-object_test.go @@ -130,7 +130,7 @@ func TestGetUserMetaHeadersMap(t *testing.T) { "x-amz-grant-write": "test@exo.ch", } - destInfo, _ := NewDestinationInfo("bucket", "object", nil, userMetadata) + destInfo, _ := NewDestinationInfo("bucket", "object", DestInfoOptions{UserMeta: userMetadata}) r := destInfo.getUserMetaHeadersMap(true) diff --git a/api-datatypes.go b/api-datatypes.go index 9db73736a..d4dbfe7c4 100644 --- a/api-datatypes.go +++ b/api-datatypes.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -62,6 +62,20 @@ func (m *StringMap) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { return nil } +// Owner name. +type Owner struct { + DisplayName string `json:"name"` + ID string `json:"id"` +} + +// UploadInfo contains information about the +// newly uploaded or copied object. +type UploadInfo struct { + ETag string + VersionID string + Size int64 +} + // ObjectInfo container for object metadata. type ObjectInfo struct { // An ETag is optionally set to md5sum of an object. In case of multipart objects, @@ -86,10 +100,7 @@ type ObjectInfo struct { UserTags map[string]string `json:"userTags"` // Owner name. - Owner struct { - DisplayName string `json:"name"` - ID string `json:"id"` - } `json:"owner"` + Owner Owner // ACL grant. Grant []struct { @@ -104,6 +115,11 @@ type ObjectInfo struct { // The class of storage used to store the object. StorageClass string `json:"storageClass"` + // Versioning related information + IsLatest bool + IsDeleteMarker bool + VersionID string `xml:"VersionId"` + // Error Err error `json:"-"` } diff --git a/api-get-object.go b/api-get-object.go index a3218f3b4..746122f01 100644 --- a/api-get-object.go +++ b/api-get-object.go @@ -23,6 +23,7 @@ import ( "fmt" "io" "net/http" + "net/url" "sync" "github.com/minio/minio-go/v7/pkg/s3utils" @@ -612,10 +613,16 @@ func (c Client) getObject(ctx context.Context, bucketName, objectName string, op return nil, ObjectInfo{}, nil, err } + urlValues := make(url.Values) + if opts.VersionID != "" { + urlValues.Set("versionId", opts.VersionID) + } + // Execute GET on objectName. resp, err := c.executeMethod(ctx, "GET", requestMetadata{ bucketName: bucketName, objectName: objectName, + queryValues: urlValues, customHeader: opts.Header(), contentSHA256Hex: emptySHA256Hex, }) diff --git a/api-get-options.go b/api-get-options.go index b169186e3..6271f2e37 100644 --- a/api-get-options.go +++ b/api-get-options.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,6 +30,7 @@ import ( type GetObjectOptions struct { headers map[string]string ServerSideEncryption encrypt.ServerSide + VersionID string } // StatObjectOptions are used to specify additional headers or options diff --git a/api-list.go b/api-list.go index 709d3561a..a683f8150 100644 --- a/api-list.go +++ b/api-list.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2019 MinIO, Inc. + * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -58,44 +58,7 @@ func (c Client) ListBuckets(ctx context.Context) ([]BucketInfo, error) { /// Bucket Read Operations. -// ListObjectsV2WithMetadata lists all objects matching the objectPrefix -// from the specified bucket. If recursion is enabled it would list -// all subdirectories and all its contents. This call adds -// UserMetadata information as well for each object. -// -// This is a MinIO extension, this will not work against other S3 -// compatible object storage vendors. -// -// Your input parameters are just bucketName, objectPrefix, recursive -// and a done channel for pro-actively closing the internal go -// routine. If you enable recursive as 'true' this function will -// return back all the objects in a given bucket name and object -// prefix. -// -// api := client.New(....) -// // Create a done channel. -// doneCh := make(chan struct{}) -// defer close(doneCh) -// // Recursively list all objects in 'mytestbucket' -// recursive := true -// // Add metadata -// metadata := true -// for message := range api.ListObjectsV2WithMetadata(ctx, "mytestbucket", "starthere", recursive, doneCh) { -// fmt.Println(message) -// } -// -func (c Client) ListObjectsV2WithMetadata(ctx context.Context, bucketName, objectPrefix string, - recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { - // Check whether this is snowball region, if yes ListObjectsV2 doesn't work, fallback to listObjectsV1. - if location, ok := c.bucketLocCache.Get(bucketName); ok { - if location == "snowball" { - return c.ListObjects(ctx, bucketName, objectPrefix, recursive, doneCh) - } - } - return c.listObjectsV2(ctx, bucketName, objectPrefix, recursive, true, doneCh) -} - -func (c Client) listObjectsV2(ctx context.Context, bucketName, objectPrefix string, recursive, metadata bool, doneCh <-chan struct{}) <-chan ObjectInfo { +func (c Client) listObjectsV2(ctx context.Context, bucketName, objectPrefix, marker string, recursive, metadata bool, maxKeys int) <-chan ObjectInfo { // Allocate new list objects channel. objectStatCh := make(chan ObjectInfo, 1) // Default listing is delimited at "/" @@ -134,7 +97,7 @@ func (c Client) listObjectsV2(ctx context.Context, bucketName, objectPrefix stri for { // Get list of objects a maximum of 1000 per request. result, err := c.listObjectsV2Query(ctx, bucketName, objectPrefix, continuationToken, - fetchOwner, metadata, delimiter, 0, "") + fetchOwner, metadata, delimiter, maxKeys, marker) if err != nil { objectStatCh <- ObjectInfo{ Err: err, @@ -149,7 +112,7 @@ func (c Client) listObjectsV2(ctx context.Context, bucketName, objectPrefix stri // Send object content. case objectStatCh <- object: // If receives done from the caller, return here. - case <-doneCh: + case <-ctx.Done(): return } } @@ -161,7 +124,7 @@ func (c Client) listObjectsV2(ctx context.Context, bucketName, objectPrefix stri // Send object prefixes. case objectStatCh <- ObjectInfo{Key: obj.Prefix}: // If receives done from the caller, return here. - case <-doneCh: + case <-ctx.Done(): return } } @@ -180,36 +143,6 @@ func (c Client) listObjectsV2(ctx context.Context, bucketName, objectPrefix stri return objectStatCh } -// ListObjectsV2 lists all objects matching the objectPrefix from -// the specified bucket. If recursion is enabled it would list -// all subdirectories and all its contents. -// -// Your input parameters are just bucketName, objectPrefix, recursive -// and a done channel for pro-actively closing the internal go -// routine. If you enable recursive as 'true' this function will -// return back all the objects in a given bucket name and object -// prefix. -// -// api := client.New(....) -// // Create a done channel. -// doneCh := make(chan struct{}) -// defer close(doneCh) -// // Recursively list all objects in 'mytestbucket' -// recursive := true -// for message := range api.ListObjectsV2(ctx, "mytestbucket", "starthere", recursive, doneCh) { -// fmt.Println(message) -// } -// -func (c Client) ListObjectsV2(ctx context.Context, bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { - // Check whether this is snowball region, if yes ListObjectsV2 doesn't work, fallback to listObjectsV1. - if location, ok := c.bucketLocCache.Get(bucketName); ok { - if location == "snowball" { - return c.ListObjects(ctx, bucketName, objectPrefix, recursive, doneCh) - } - } - return c.listObjectsV2(ctx, bucketName, objectPrefix, recursive, false, doneCh) -} - // listObjectsV2Query - (List Objects V2) - List some or all (up to 1000) of the objects in a bucket. // // You can use the request parameters as selection criteria to return a subset of the objects in a bucket. @@ -319,31 +252,7 @@ func (c Client) listObjectsV2Query(ctx context.Context, bucketName, objectPrefix return listBucketResult, nil } -// ListObjects - (List Objects with context) - List some -// objects or all recursively. -// ListObjects lists all objects matching the objectPrefix from -// the specified bucket. If recursion is enabled it would list -// all subdirectories and all its contents. -// -// Your input parameters are just bucketName, objectPrefix, recursive -// and a done channel for pro-actively closing the internal go -// routine. If you enable recursive as 'true' this function will -// return back all the objects in a given bucket name and object -// prefix. -// -// api := client.New(....) -// // Create a done channel. -// doneCh := make(chan struct{}) -// defer close(doneCh) -// // Recurively list all objects in 'mytestbucket' -// recursive := true -// for message := range api.ListObjects(ctx, "mytestbucket", -// "starthere", recursive, doneCh) { -// fmt.Println(message) -// } -// -func (c Client) ListObjects(ctx context.Context, bucketName, objectPrefix string, recursive bool, - doneCh <-chan struct{}) <-chan ObjectInfo { +func (c Client) listObjects(ctx context.Context, bucketName, objectPrefix, marker string, recursive bool, maxKeys int) <-chan ObjectInfo { // Allocate new list objects channel. objectStatCh := make(chan ObjectInfo, 1) // Default listing is delimited at "/" @@ -372,11 +281,10 @@ func (c Client) ListObjects(ctx context.Context, bucketName, objectPrefix string // Initiate list objects goroutine here. go func(objectStatCh chan<- ObjectInfo) { defer close(objectStatCh) - // Save marker for next request. - var marker string + for { // Get list of objects a maximum of 1000 per request. - result, err := c.listObjectsQuery(ctx, bucketName, objectPrefix, marker, delimiter, 0) + result, err := c.listObjectsQuery(ctx, bucketName, objectPrefix, marker, delimiter, maxKeys) if err != nil { objectStatCh <- ObjectInfo{ Err: err, @@ -392,7 +300,7 @@ func (c Client) ListObjects(ctx context.Context, bucketName, objectPrefix string // Send object content. case objectStatCh <- object: // If receives done from the caller, return here. - case <-doneCh: + case <-ctx.Done(): return } } @@ -404,7 +312,7 @@ func (c Client) ListObjects(ctx context.Context, bucketName, objectPrefix string // Send object prefixes. case objectStatCh <- ObjectInfo{Key: obj.Prefix}: // If receives done from the caller, return here. - case <-doneCh: + case <-ctx.Done(): return } } @@ -423,6 +331,205 @@ func (c Client) ListObjects(ctx context.Context, bucketName, objectPrefix string return objectStatCh } +func (c Client) listObjectVersions(ctx context.Context, bucketName, prefix, marker string, recursive bool, maxKeys int) <-chan ObjectInfo { + // Allocate new list objects channel. + resultCh := make(chan ObjectInfo, 1) + // Default listing is delimited at "/" + delimiter := "/" + if recursive { + // If recursive we do not delimit. + delimiter = "" + } + + // Validate bucket name. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + defer close(resultCh) + resultCh <- ObjectInfo{ + Err: err, + } + return resultCh + } + + // Validate incoming object prefix. + if err := s3utils.CheckValidObjectNamePrefix(prefix); err != nil { + defer close(resultCh) + resultCh <- ObjectInfo{ + Err: err, + } + return resultCh + } + + // Initiate list objects goroutine here. + go func(resultCh chan<- ObjectInfo) { + defer close(resultCh) + + var ( + keyMarker = marker + versionIDMarker = "" + ) + + for { + // Get list of objects a maximum of 1000 per request. + result, err := c.listObjectVersionsQuery(ctx, bucketName, prefix, keyMarker, versionIDMarker, delimiter, maxKeys) + if err != nil { + resultCh <- ObjectInfo{ + Err: err, + } + return + } + + // If contents are available loop through and send over channel. + for _, version := range result.Versions { + info := ObjectInfo{ + ETag: trimEtag(version.ETag), + Key: version.Key, + LastModified: version.LastModified, + Size: version.Size, + Owner: version.Owner, + StorageClass: version.StorageClass, + IsLatest: version.IsLatest, + VersionID: version.VersionID, + + IsDeleteMarker: version.isDeleteMarker, + } + select { + // Send object version info. + case resultCh <- info: + // If receives done from the caller, return here. + case <-ctx.Done(): + return + } + } + + // Send all common prefixes if any. + // NOTE: prefixes are only present if the request is delimited. + for _, obj := range result.CommonPrefixes { + select { + // Send object prefixes. + case resultCh <- ObjectInfo{Key: obj.Prefix}: + // If receives done from the caller, return here. + case <-ctx.Done(): + return + } + } + + // If next key marker is present, save it for next request. + if result.NextKeyMarker != "" { + keyMarker = result.NextKeyMarker + } + + // If next version id marker is present, save it for next request. + if result.NextVersionIDMarker != "" { + versionIDMarker = result.NextVersionIDMarker + } + + // Listing ends result is not truncated, return right here. + if !result.IsTruncated { + return + } + } + }(resultCh) + return resultCh +} + +// listObjectVersions - (List Object Versions) - List some or all (up to 1000) of the existing objects +// and their versions in a bucket. +// +// You can use the request parameters as selection criteria to return a subset of the objects in a bucket. +// request parameters :- +// --------- +// ?key-marker - Specifies the key to start with when listing objects in a bucket. +// ?version-id-marker - Specifies the version id marker to start with when listing objects with versions in a bucket. +// ?delimiter - A delimiter is a character you use to group keys. +// ?prefix - Limits the response to keys that begin with the specified prefix. +// ?max-keys - Sets the maximum number of keys returned in the response body. +func (c Client) listObjectVersionsQuery(ctx context.Context, bucketName, prefix, keyMarker, versionIDMarker, delimiter string, maxkeys int) (ListVersionsResult, error) { + // Validate bucket name. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return ListVersionsResult{}, err + } + // Validate object prefix. + if err := s3utils.CheckValidObjectNamePrefix(prefix); err != nil { + return ListVersionsResult{}, err + } + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + + // Set versions to trigger versioning API + urlValues.Set("versions", "") + + // Set object prefix, prefix value to be set to empty is okay. + urlValues.Set("prefix", prefix) + + // Set delimiter, delimiter value to be set to empty is okay. + urlValues.Set("delimiter", delimiter) + + // Set object marker. + if keyMarker != "" { + urlValues.Set("key-marker", keyMarker) + } + + // Set max keys. + if maxkeys > 0 { + urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys)) + } + + // Set version ID marker + if versionIDMarker != "" { + urlValues.Set("version-id-marker", versionIDMarker) + } + + // Always set encoding-type + urlValues.Set("encoding-type", "url") + + // Execute GET on bucket to list objects. + resp, err := c.executeMethod(ctx, "GET", requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return ListVersionsResult{}, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return ListVersionsResult{}, httpRespToErrorResponse(resp, bucketName, "") + } + } + + // Decode ListVersionsResult XML. + listObjectVersionsOutput := ListVersionsResult{} + err = xmlDecoder(resp.Body, &listObjectVersionsOutput) + if err != nil { + return ListVersionsResult{}, err + } + + for i, obj := range listObjectVersionsOutput.Versions { + listObjectVersionsOutput.Versions[i].Key, err = decodeS3Name(obj.Key, listObjectVersionsOutput.EncodingType) + if err != nil { + return listObjectVersionsOutput, err + } + } + + for i, obj := range listObjectVersionsOutput.CommonPrefixes { + listObjectVersionsOutput.CommonPrefixes[i].Prefix, err = decodeS3Name(obj.Prefix, listObjectVersionsOutput.EncodingType) + if err != nil { + return listObjectVersionsOutput, err + } + } + + if listObjectVersionsOutput.NextKeyMarker != "" { + listObjectVersionsOutput.NextKeyMarker, err = decodeS3Name(listObjectVersionsOutput.NextKeyMarker, listObjectVersionsOutput.EncodingType) + if err != nil { + return listObjectVersionsOutput, err + } + } + + return listObjectVersionsOutput, nil +} + // listObjects - (List Objects) - List some or all (up to 1000) of the objects in a bucket. // // You can use the request parameters as selection criteria to return a subset of the objects in a bucket. @@ -510,6 +617,50 @@ func (c Client) listObjectsQuery(ctx context.Context, bucketName, objectPrefix, return listBucketResult, nil } +// ListObjectsOptions holds all options of a list object request +type ListObjectsOptions struct { + // Force the deprecated listing V1 + ForceV1 bool + // Include objects versions in the listing + WithVersions bool + // Include objects metadata in the listing + WithMetadata bool + // Only list objects with the prefix + Prefix string + // Ignore '/' delimiter + Recursive bool + // Start listing after the specified path + StartAfter string + // The maximum number of returned objects per request + MaxKeys int +} + +// ListObjects returns objects list after evaluating the passed options. +// +// api := client.New(....) +// for object := range api.ListObjects(ctx, "mytestbucket", minio.ListObjectsOptions{Prefix: "starthere", Recursive:true}) { +// fmt.Println(object) +// } +// +func (c Client) ListObjects(ctx context.Context, bucketName string, opts ListObjectsOptions) <-chan ObjectInfo { + if opts.WithVersions { + return c.listObjectVersions(ctx, bucketName, opts.Prefix, opts.StartAfter, opts.Recursive, opts.MaxKeys) + } + + if opts.ForceV1 { + return c.listObjects(ctx, bucketName, opts.Prefix, opts.StartAfter, opts.Recursive, opts.MaxKeys) + } + + // Check whether this is snowball region, if yes ListObjectsV2 doesn't work, fallback to listObjectsV1. + if location, ok := c.bucketLocCache.Get(bucketName); ok { + if location == "snowball" { + return c.listObjects(ctx, bucketName, opts.Prefix, opts.StartAfter, opts.Recursive, opts.MaxKeys) + } + } + + return c.listObjectsV2(ctx, bucketName, opts.Prefix, opts.StartAfter, opts.Recursive, opts.WithMetadata, opts.MaxKeys) +} + // ListIncompleteUploads - List incompletely uploaded multipart objects. // // ListIncompleteUploads lists all incompleted objects matching the diff --git a/api-object-tagging.go b/api-object-tagging.go index ec8a161cb..4152366b4 100644 --- a/api-object-tagging.go +++ b/api-object-tagging.go @@ -28,9 +28,15 @@ import ( "github.com/minio/minio-go/v7/pkg/tags" ) -// PutObjectTagging replaces or creates object tag(s) with a context to control cancellations -// and timeouts. -func (c Client) PutObjectTagging(ctx context.Context, bucketName, objectName string, objectTags map[string]string) error { +// PutObjectTaggingOptions holds an object version id +// to update tag(s) of a specific object version +type PutObjectTaggingOptions struct { + VersionID string +} + +// PutObjectTagging replaces or creates object tag(s) and can target +// a specific object version in a versioned bucket. +func (c Client) PutObjectTagging(ctx context.Context, bucketName, objectName string, objectTags map[string]string, opts PutObjectTaggingOptions) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -41,6 +47,10 @@ func (c Client) PutObjectTagging(ctx context.Context, bucketName, objectName str urlValues := make(url.Values) urlValues.Set("tagging", "") + if opts.VersionID != "" { + urlValues.Set("versionId", opts.VersionID) + } + tags, err := tags.NewTags(objectTags, true) if err != nil { return err @@ -74,14 +84,24 @@ func (c Client) PutObjectTagging(ctx context.Context, bucketName, objectName str return nil } -// GetObjectTagging fetches object tag(s) with a context to control cancellations -// and timeouts. -func (c Client) GetObjectTagging(ctx context.Context, bucketName, objectName string) (map[string]string, error) { +// GetObjectTaggingOptions holds the object version ID +// to fetch the tagging key/value pairs +type GetObjectTaggingOptions struct { + VersionID string +} + +// GetObjectTagging fetches object tag(s) with options to target +// a specific object version in a versioned bucket. +func (c Client) GetObjectTagging(ctx context.Context, bucketName, objectName string, opts GetObjectTaggingOptions) (map[string]string, error) { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("tagging", "") + if opts.VersionID != "" { + urlValues.Set("versionId", opts.VersionID) + } + // Execute GET on object to get object tag(s) resp, err := c.executeMethod(ctx, "GET", requestMetadata{ bucketName: bucketName, @@ -108,14 +128,23 @@ func (c Client) GetObjectTagging(ctx context.Context, bucketName, objectName str return tags.ToMap(), err } -// RemoveObjectTagging removes object tag(s) with a context to control cancellations -// and timeouts. -func (c Client) RemoveObjectTagging(ctx context.Context, bucketName, objectName string) error { +// RemoveObjectTaggingOptions holds the version id of the object to remove +type RemoveObjectTaggingOptions struct { + VersionID string +} + +// RemoveObjectTagging removes object tag(s) with options to control a specific object +// version in a versioned bucket +func (c Client) RemoveObjectTagging(ctx context.Context, bucketName, objectName string, opts RemoveObjectTaggingOptions) error { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("tagging", "") + if opts.VersionID != "" { + urlValues.Set("versionId", opts.VersionID) + } + // Execute DELETE on object to remove object tag(s) resp, err := c.executeMethod(ctx, "DELETE", requestMetadata{ bucketName: bucketName, diff --git a/api-put-object-copy.go b/api-put-object-copy.go index 227ddd1bb..baf724760 100644 --- a/api-put-object-copy.go +++ b/api-put-object-copy.go @@ -29,12 +29,12 @@ import ( ) // CopyObject - copy a source object into a new object -func (c Client) CopyObject(ctx context.Context, dst DestinationInfo, src SourceInfo) error { +func (c Client) CopyObject(ctx context.Context, dst DestinationInfo, src SourceInfo) (UploadInfo, error) { return c.CopyObjectWithProgress(ctx, dst, src, nil) } // CopyObjectWithProgress is like CopyObject with additional progress bar. -func (c Client) CopyObjectWithProgress(ctx context.Context, dst DestinationInfo, src SourceInfo, progress io.Reader) error { +func (c Client) CopyObjectWithProgress(ctx context.Context, dst DestinationInfo, src SourceInfo, progress io.Reader) (UploadInfo, error) { header := make(http.Header) for k, v := range src.Headers { header[k] = v @@ -60,7 +60,7 @@ func (c Client) CopyObjectWithProgress(ctx context.Context, dst DestinationInfo, if progress != nil { size, _, _, err = src.getProps(c) if err != nil { - return err + return UploadInfo{}, err } } @@ -81,12 +81,12 @@ func (c Client) CopyObjectWithProgress(ctx context.Context, dst DestinationInfo, customHeader: header, }) if err != nil { - return err + return UploadInfo{}, err } defer closeResponse(resp) if resp.StatusCode != http.StatusOK { - return httpRespToErrorResponse(resp, dst.bucket, dst.object) + return UploadInfo{}, httpRespToErrorResponse(resp, dst.bucket, dst.object) } // Update the progress properly after successful copy. @@ -94,5 +94,9 @@ func (c Client) CopyObjectWithProgress(ctx context.Context, dst DestinationInfo, io.CopyN(ioutil.Discard, progress, size) } - return nil + return UploadInfo{ + VersionID: resp.Header.Get("x-amz-version-id"), + Size: size, + ETag: trimEtag(resp.Header.Get("ETag")), + }, nil } diff --git a/api-put-object-file-context.go b/api-put-object-file-context.go index a55da8364..6c0f20df3 100644 --- a/api-put-object-file-context.go +++ b/api-put-object-file-context.go @@ -27,27 +27,27 @@ import ( ) // FPutObject - Create an object in a bucket, with contents from file at filePath. Allows request cancellation. -func (c Client) FPutObject(ctx context.Context, bucketName, objectName, filePath string, opts PutObjectOptions) (n int64, err error) { +func (c Client) FPutObject(ctx context.Context, bucketName, objectName, filePath string, opts PutObjectOptions) (info UploadInfo, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return 0, err + return UploadInfo{}, err } if err := s3utils.CheckValidObjectName(objectName); err != nil { - return 0, err + return UploadInfo{}, err } // Open the referenced file. fileReader, err := os.Open(filePath) // If any error fail quickly here. if err != nil { - return 0, err + return UploadInfo{}, err } defer fileReader.Close() // Save the file stat. fileStat, err := fileReader.Stat() if err != nil { - return 0, err + return UploadInfo{}, err } // Save the file size. diff --git a/api-put-object-multipart.go b/api-put-object-multipart.go index 29c7f1f46..978b26505 100644 --- a/api-put-object-multipart.go +++ b/api-put-object-multipart.go @@ -37,8 +37,8 @@ import ( ) func (c Client) putObjectMultipart(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, - opts PutObjectOptions) (n int64, err error) { - n, err = c.putObjectMultipartNoStream(ctx, bucketName, objectName, reader, opts) + opts PutObjectOptions) (info UploadInfo, err error) { + info, err = c.putObjectMultipartNoStream(ctx, bucketName, objectName, reader, opts) if err != nil { errResp := ToErrorResponse(err) // Verify if multipart functionality is not available, if not @@ -46,22 +46,22 @@ func (c Client) putObjectMultipart(ctx context.Context, bucketName, objectName s if errResp.Code == "AccessDenied" && strings.Contains(errResp.Message, "Access Denied") { // Verify if size of reader is greater than '5GiB'. if size > maxSinglePutObjectSize { - return 0, errEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) + return UploadInfo{}, errEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) } // Fall back to uploading as single PutObject operation. return c.putObject(ctx, bucketName, objectName, reader, size, opts) } } - return n, err + return info, err } -func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, objectName string, reader io.Reader, opts PutObjectOptions) (n int64, err error) { +func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, objectName string, reader io.Reader, opts PutObjectOptions) (info UploadInfo, err error) { // Input validation. if err = s3utils.CheckValidBucketName(bucketName); err != nil { - return 0, err + return UploadInfo{}, err } if err = s3utils.CheckValidObjectName(objectName); err != nil { - return 0, err + return UploadInfo{}, err } // Total data read and written to server. should be equal to @@ -74,13 +74,13 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje // Calculate the optimal parts info for a given size. totalPartsCount, partSize, _, err := optimalPartInfo(-1, opts.PartSize) if err != nil { - return 0, err + return UploadInfo{}, err } // Initiate a new multipart upload. uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts) if err != nil { - return 0, err + return UploadInfo{}, err } defer func() { @@ -109,7 +109,7 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje break } if rErr != nil && rErr != io.ErrUnexpectedEOF { - return 0, rErr + return UploadInfo{}, rErr } // Calculates hash sums while copying partSize bytes into cw. @@ -139,7 +139,7 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje objPart, uerr := c.uploadPart(ctx, bucketName, objectName, uploadID, rd, partNumber, md5Base64, sha256Hex, int64(length), opts.ServerSideEncryption) if uerr != nil { - return totalUploadedSize, uerr + return UploadInfo{}, uerr } // Save successfully uploaded part metadata. @@ -163,7 +163,7 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje for i := 1; i < partNumber; i++ { part, ok := partsInfo[i] if !ok { - return 0, errInvalidArgument(fmt.Sprintf("Missing part number %d", i)) + return UploadInfo{}, errInvalidArgument(fmt.Sprintf("Missing part number %d", i)) } complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ ETag: part.ETag, @@ -173,12 +173,14 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje // Sort all completed parts. sort.Sort(completedParts(complMultipartUpload.Parts)) - if _, err = c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload); err != nil { - return totalUploadedSize, err + + uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload) + if err != nil { + return UploadInfo{}, err } - // Return final size. - return totalUploadedSize, nil + uploadInfo.Size = totalUploadedSize + return uploadInfo, nil } // initiateMultipartUpload - Initiates a multipart upload and returns an upload ID. @@ -298,13 +300,13 @@ func (c Client) uploadPart(ctx context.Context, bucketName, objectName, uploadID // completeMultipartUpload - Completes a multipart upload by assembling previously uploaded parts. func (c Client) completeMultipartUpload(ctx context.Context, bucketName, objectName, uploadID string, - complete completeMultipartUpload) (completeMultipartUploadResult, error) { + complete completeMultipartUpload) (UploadInfo, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return completeMultipartUploadResult{}, err + return UploadInfo{}, err } if err := s3utils.CheckValidObjectName(objectName); err != nil { - return completeMultipartUploadResult{}, err + return UploadInfo{}, err } // Initialize url queries. @@ -313,7 +315,7 @@ func (c Client) completeMultipartUpload(ctx context.Context, bucketName, objectN // Marshal complete multipart body. completeMultipartUploadBytes, err := xml.Marshal(complete) if err != nil { - return completeMultipartUploadResult{}, err + return UploadInfo{}, err } // Instantiate all the complete multipart buffer. @@ -331,11 +333,11 @@ func (c Client) completeMultipartUpload(ctx context.Context, bucketName, objectN resp, err := c.executeMethod(ctx, "POST", reqMetadata) defer closeResponse(resp) if err != nil { - return completeMultipartUploadResult{}, err + return UploadInfo{}, err } if resp != nil { if resp.StatusCode != http.StatusOK { - return completeMultipartUploadResult{}, httpRespToErrorResponse(resp, bucketName, objectName) + return UploadInfo{}, httpRespToErrorResponse(resp, bucketName, objectName) } } @@ -343,14 +345,14 @@ func (c Client) completeMultipartUpload(ctx context.Context, bucketName, objectN var b []byte b, err = ioutil.ReadAll(resp.Body) if err != nil { - return completeMultipartUploadResult{}, err + return UploadInfo{}, err } // Decode completed multipart upload response on success. completeMultipartUploadResult := completeMultipartUploadResult{} err = xmlDecoder(bytes.NewReader(b), &completeMultipartUploadResult) if err != nil { // xml parsing failure due to presence an ill-formed xml fragment - return completeMultipartUploadResult, err + return UploadInfo{}, err } else if completeMultipartUploadResult.Bucket == "" { // xml's Decode method ignores well-formed xml that don't apply to the type of value supplied. // In this case, it would leave completeMultipartUploadResult with the corresponding zero-values @@ -361,9 +363,14 @@ func (c Client) completeMultipartUpload(ctx context.Context, bucketName, objectN err = xmlDecoder(bytes.NewReader(b), &completeMultipartUploadErr) if err != nil { // xml parsing failure due to presence an ill-formed xml fragment - return completeMultipartUploadResult, err + return UploadInfo{}, err } - return completeMultipartUploadResult, completeMultipartUploadErr + return UploadInfo{}, completeMultipartUploadErr } - return completeMultipartUploadResult, nil + + return UploadInfo{ + ETag: trimEtag(completeMultipartUploadResult.ETag), + VersionID: resp.Header.Get("x-amz-version-id"), + }, nil + } diff --git a/api-put-object-streaming.go b/api-put-object-streaming.go index 657fd9fb3..4c8393b62 100644 --- a/api-put-object-streaming.go +++ b/api-put-object-streaming.go @@ -41,13 +41,13 @@ import ( // - Any reader which has a method 'ReadAt()' // func (c Client) putObjectMultipartStream(ctx context.Context, bucketName, objectName string, - reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) { + reader io.Reader, size int64, opts PutObjectOptions) (info UploadInfo, err error) { if !isObject(reader) && isReadAt(reader) && !opts.SendContentMd5 { // Verify if the reader implements ReadAt and it is not a *minio.Object then we will use parallel uploader. - n, err = c.putObjectMultipartStreamFromReadAt(ctx, bucketName, objectName, reader.(io.ReaderAt), size, opts) + info, err = c.putObjectMultipartStreamFromReadAt(ctx, bucketName, objectName, reader.(io.ReaderAt), size, opts) } else { - n, err = c.putObjectMultipartStreamOptionalChecksum(ctx, bucketName, objectName, reader, size, opts) + info, err = c.putObjectMultipartStreamOptionalChecksum(ctx, bucketName, objectName, reader, size, opts) } if err != nil { errResp := ToErrorResponse(err) @@ -56,13 +56,13 @@ func (c Client) putObjectMultipartStream(ctx context.Context, bucketName, object if errResp.Code == "AccessDenied" && strings.Contains(errResp.Message, "Access Denied") { // Verify if size of reader is greater than '5GiB'. if size > maxSinglePutObjectSize { - return 0, errEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) + return UploadInfo{}, errEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) } // Fall back to uploading as single PutObject operation. return c.putObject(ctx, bucketName, objectName, reader, size, opts) } } - return n, err + return info, err } // uploadedPartRes - the response received from a part upload. @@ -90,25 +90,25 @@ type uploadPartReq struct { // cleaned automatically when the caller i.e http client closes the // stream after uploading all the contents successfully. func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketName, objectName string, - reader io.ReaderAt, size int64, opts PutObjectOptions) (n int64, err error) { + reader io.ReaderAt, size int64, opts PutObjectOptions) (info UploadInfo, err error) { // Input validation. if err = s3utils.CheckValidBucketName(bucketName); err != nil { - return 0, err + return UploadInfo{}, err } if err = s3utils.CheckValidObjectName(objectName); err != nil { - return 0, err + return UploadInfo{}, err } // Calculate the optimal parts info for a given size. totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(size, opts.PartSize) if err != nil { - return 0, err + return UploadInfo{}, err } // Initiate a new multipart upload. uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts) if err != nil { - return 0, err + return UploadInfo{}, err } // Aborts the multipart upload in progress, if the @@ -196,7 +196,7 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa for u := 1; u <= totalPartsCount; u++ { uploadRes := <-uploadedPartsCh if uploadRes.Error != nil { - return totalUploadedSize, uploadRes.Error + return UploadInfo{}, uploadRes.Error } // Update the totalUploadedSize. totalUploadedSize += uploadRes.Size @@ -209,39 +209,40 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa // Verify if we uploaded all the data. if totalUploadedSize != size { - return totalUploadedSize, errUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) + return UploadInfo{}, errUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) } // Sort all completed parts. sort.Sort(completedParts(complMultipartUpload.Parts)) - _, err = c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload) + + uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload) if err != nil { - return totalUploadedSize, err + return UploadInfo{}, err } - // Return final size. - return totalUploadedSize, nil + uploadInfo.Size = totalUploadedSize + return uploadInfo, nil } func (c Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, bucketName, objectName string, - reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) { + reader io.Reader, size int64, opts PutObjectOptions) (info UploadInfo, err error) { // Input validation. if err = s3utils.CheckValidBucketName(bucketName); err != nil { - return 0, err + return UploadInfo{}, err } if err = s3utils.CheckValidObjectName(objectName); err != nil { - return 0, err + return UploadInfo{}, err } // Calculate the optimal parts info for a given size. totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(size, opts.PartSize) if err != nil { - return 0, err + return UploadInfo{}, err } // Initiates a new multipart request uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts) if err != nil { - return 0, err + return UploadInfo{}, err } // Aborts the multipart upload if the function returns @@ -282,7 +283,7 @@ func (c Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, bu break } if rerr != nil && rerr != io.ErrUnexpectedEOF && rerr != io.EOF { - return 0, rerr + return UploadInfo{}, rerr } // Calculate md5sum. hash := c.md5Hasher() @@ -303,7 +304,7 @@ func (c Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, bu io.LimitReader(hookReader, partSize), partNumber, md5Base64, "", partSize, opts.ServerSideEncryption) if uerr != nil { - return totalUploadedSize, uerr + return UploadInfo{}, uerr } // Save successfully uploaded part metadata. @@ -316,7 +317,7 @@ func (c Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, bu // Verify if we uploaded all the data. if size > 0 { if totalUploadedSize != size { - return totalUploadedSize, errUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) + return UploadInfo{}, errUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) } } @@ -328,7 +329,7 @@ func (c Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, bu for i := 1; i < partNumber; i++ { part, ok := partsInfo[i] if !ok { - return 0, errInvalidArgument(fmt.Sprintf("Missing part number %d", i)) + return UploadInfo{}, errInvalidArgument(fmt.Sprintf("Missing part number %d", i)) } complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ ETag: part.ETag, @@ -338,34 +339,35 @@ func (c Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, bu // Sort all completed parts. sort.Sort(completedParts(complMultipartUpload.Parts)) - _, err = c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload) + + uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload) if err != nil { - return totalUploadedSize, err + return UploadInfo{}, err } - // Return final size. - return totalUploadedSize, nil + uploadInfo.Size = totalUploadedSize + return uploadInfo, nil } // putObject special function used Google Cloud Storage. This special function // is used for Google Cloud Storage since Google's multipart API is not S3 compatible. -func (c Client) putObject(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) { +func (c Client) putObject(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions) (info UploadInfo, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return 0, err + return UploadInfo{}, err } if err := s3utils.CheckValidObjectName(objectName); err != nil { - return 0, err + return UploadInfo{}, err } // Size -1 is only supported on Google Cloud Storage, we error // out in all other situations. if size < 0 && !s3utils.IsGoogleEndpoint(*c.endpointURL) { - return 0, errEntityTooSmall(size, bucketName, objectName) + return UploadInfo{}, errEntityTooSmall(size, bucketName, objectName) } if opts.SendContentMd5 && s3utils.IsGoogleEndpoint(*c.endpointURL) && size < 0 { - return 0, errInvalidArgument("MD5Sum cannot be calculated with size '-1'") + return UploadInfo{}, errInvalidArgument("MD5Sum cannot be calculated with size '-1'") } if size > 0 { @@ -374,7 +376,7 @@ func (c Client) putObject(ctx context.Context, bucketName, objectName string, re if ok { offset, err := seeker.Seek(0, io.SeekCurrent) if err != nil { - return 0, errInvalidArgument(err.Error()) + return UploadInfo{}, errInvalidArgument(err.Error()) } reader = io.NewSectionReader(reader.(io.ReaderAt), offset, size) } @@ -388,7 +390,7 @@ func (c Client) putObject(ctx context.Context, bucketName, objectName string, re length, rErr := io.ReadFull(reader, buf) if rErr != nil && rErr != io.ErrUnexpectedEOF { - return 0, rErr + return UploadInfo{}, rErr } // Calculate md5sum. @@ -405,25 +407,18 @@ func (c Client) putObject(ctx context.Context, bucketName, objectName string, re // This function does not calculate sha256 and md5sum for payload. // Execute put object. - st, err := c.putObjectDo(ctx, bucketName, objectName, readSeeker, md5Base64, "", size, opts) - if err != nil { - return 0, err - } - if st.Size != size { - return 0, errUnexpectedEOF(st.Size, size, bucketName, objectName) - } - return size, nil + return c.putObjectDo(ctx, bucketName, objectName, readSeeker, md5Base64, "", size, opts) } // putObjectDo - executes the put object http operation. // NOTE: You must have WRITE permissions on a bucket to add an object to it. -func (c Client) putObjectDo(ctx context.Context, bucketName, objectName string, reader io.Reader, md5Base64, sha256Hex string, size int64, opts PutObjectOptions) (ObjectInfo, error) { +func (c Client) putObjectDo(ctx context.Context, bucketName, objectName string, reader io.Reader, md5Base64, sha256Hex string, size int64, opts PutObjectOptions) (UploadInfo, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return ObjectInfo{}, err + return UploadInfo{}, err } if err := s3utils.CheckValidObjectName(objectName); err != nil { - return ObjectInfo{}, err + return UploadInfo{}, err } // Set headers. customHeader := opts.Header() @@ -447,20 +442,17 @@ func (c Client) putObjectDo(ctx context.Context, bucketName, objectName string, resp, err := c.executeMethod(ctx, "PUT", reqMetadata) defer closeResponse(resp) if err != nil { - return ObjectInfo{}, err + return UploadInfo{}, err } if resp != nil { if resp.StatusCode != http.StatusOK { - return ObjectInfo{}, httpRespToErrorResponse(resp, bucketName, objectName) + return UploadInfo{}, httpRespToErrorResponse(resp, bucketName, objectName) } } - var objInfo ObjectInfo - // Trim off the odd double quotes from ETag in the beginning and end. - objInfo.ETag = trimEtag(resp.Header.Get("ETag")) - // A success here means data was written to server successfully. - objInfo.Size = size - - // Return here. - return objInfo, nil + return UploadInfo{ + ETag: trimEtag(resp.Header.Get("ETag")), + VersionID: resp.Header.Get("x-amz-version-id"), + Size: size, + }, nil } diff --git a/api-put-object.go b/api-put-object.go index 4c8dd2829..6b7b6e820 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -174,23 +174,23 @@ func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].Part // until input stream reaches EOF. Maximum object size that can // be uploaded through this operation will be 5TiB. func (c Client) PutObject(ctx context.Context, bucketName, objectName string, reader io.Reader, objectSize int64, - opts PutObjectOptions) (n int64, err error) { + opts PutObjectOptions) (info UploadInfo, err error) { if objectSize < 0 && opts.DisableMultipart { - return 0, errors.New("object size must be provided with disable multipart upload") + return UploadInfo{}, errors.New("object size must be provided with disable multipart upload") } err = opts.validate() if err != nil { - return 0, err + return UploadInfo{}, err } return c.putObjectCommon(ctx, bucketName, objectName, reader, objectSize, opts) } -func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) { +func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions) (info UploadInfo, err error) { // Check for largest object size allowed. if size > int64(maxMultipartPutObjectSize) { - return 0, errEntityTooLarge(size, maxMultipartPutObjectSize, bucketName, objectName) + return UploadInfo{}, errEntityTooLarge(size, maxMultipartPutObjectSize, bucketName, objectName) } // NOTE: Streaming signature is not supported by GCS. @@ -220,13 +220,13 @@ func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName stri return c.putObjectMultipartStream(ctx, bucketName, objectName, reader, size, opts) } -func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName, objectName string, reader io.Reader, opts PutObjectOptions) (n int64, err error) { +func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName, objectName string, reader io.Reader, opts PutObjectOptions) (info UploadInfo, err error) { // Input validation. if err = s3utils.CheckValidBucketName(bucketName); err != nil { - return 0, err + return UploadInfo{}, err } if err = s3utils.CheckValidObjectName(objectName); err != nil { - return 0, err + return UploadInfo{}, err } // Total data read and written to server. should be equal to @@ -239,12 +239,12 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName // Calculate the optimal parts info for a given size. totalPartsCount, partSize, _, err := optimalPartInfo(-1, opts.PartSize) if err != nil { - return 0, err + return UploadInfo{}, err } // Initiate a new multipart upload. uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts) if err != nil { - return 0, err + return UploadInfo{}, err } defer func() { @@ -268,7 +268,7 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName break } if rerr != nil && rerr != io.ErrUnexpectedEOF && rerr != io.EOF { - return 0, rerr + return UploadInfo{}, rerr } var md5Base64 string @@ -288,7 +288,7 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName objPart, uerr := c.uploadPart(ctx, bucketName, objectName, uploadID, rd, partNumber, md5Base64, "", int64(length), opts.ServerSideEncryption) if uerr != nil { - return totalUploadedSize, uerr + return UploadInfo{}, uerr } // Save successfully uploaded part metadata. @@ -312,7 +312,7 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName for i := 1; i < partNumber; i++ { part, ok := partsInfo[i] if !ok { - return 0, errInvalidArgument(fmt.Sprintf("Missing part number %d", i)) + return UploadInfo{}, errInvalidArgument(fmt.Sprintf("Missing part number %d", i)) } complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ ETag: part.ETag, @@ -322,10 +322,12 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName // Sort all completed parts. sort.Sort(completedParts(complMultipartUpload.Parts)) - if _, err = c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload); err != nil { - return totalUploadedSize, err + + uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload) + if err != nil { + return UploadInfo{}, err } - // Return final size. - return totalUploadedSize, nil + uploadInfo.Size = totalUploadedSize + return uploadInfo, nil } diff --git a/api-remove.go b/api-remove.go index df4d2f331..28e030ced 100644 --- a/api-remove.go +++ b/api-remove.go @@ -58,19 +58,14 @@ func (c Client) RemoveBucket(ctx context.Context, bucketName string) error { return nil } -// RemoveObject remove an object from a bucket. -func (c Client) RemoveObject(ctx context.Context, bucketName, objectName string) error { - return c.RemoveObjectWithOptions(ctx, bucketName, objectName, RemoveObjectOptions{}) -} - // RemoveObjectOptions represents options specified by user for RemoveObject call type RemoveObjectOptions struct { GovernanceBypass bool VersionID string } -// RemoveObjectWithOptions removes an object from a bucket. -func (c Client) RemoveObjectWithOptions(ctx context.Context, bucketName, objectName string, opts RemoveObjectOptions) error { +// RemoveObject removes an object from a bucket. +func (c Client) RemoveObject(ctx context.Context, bucketName, objectName string, opts RemoveObjectOptions) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -122,14 +117,15 @@ func (c Client) RemoveObjectWithOptions(ctx context.Context, bucketName, objectN // RemoveObjectError - container of Multi Delete S3 API error type RemoveObjectError struct { ObjectName string + VersionID string Err error } // generateRemoveMultiObjects - generate the XML request for remove multi objects request -func generateRemoveMultiObjectsRequest(objects []string) []byte { +func generateRemoveMultiObjectsRequest(objects []ObjectVersion) []byte { rmObjects := []deleteObject{} for _, obj := range objects { - rmObjects = append(rmObjects, deleteObject{Key: obj}) + rmObjects = append(rmObjects, deleteObject(obj)) } xmlBytes, _ := xml.Marshal(deleteMultiObjects{Objects: rmObjects, Quiet: true}) return xmlBytes @@ -137,7 +133,7 @@ func generateRemoveMultiObjectsRequest(objects []string) []byte { // processRemoveMultiObjectsResponse - parse the remove multi objects web service // and return the success/failure result status for each object -func processRemoveMultiObjectsResponse(body io.Reader, objects []string, errorCh chan<- RemoveObjectError) { +func processRemoveMultiObjectsResponse(body io.Reader, objects []ObjectVersion, errorCh chan<- RemoveObjectError) { // Parse multi delete XML response rmResult := &deleteMultiObjectsResult{} err := xmlDecoder(body, rmResult) @@ -158,10 +154,16 @@ func processRemoveMultiObjectsResponse(body io.Reader, objects []string, errorCh } } -// RemoveObjects removes multiple objects from a bucket. -// The list of objects to remove are received from objectsCh. -// Remove failures are sent back via error channel. -func (c Client) RemoveObjects(ctx context.Context, bucketName string, objectsCh <-chan string) <-chan RemoveObjectError { +// ObjectVersion points to a specific object version +type ObjectVersion struct { + Key string + VersionID string +} + +// RemoveObjectsWithVersions removes multiple objects from a bucket while +// it is possible to specify objects versions which are received from +// objectsCh. Remove failures are sent back via error channel. +func (c Client) RemoveObjectsWithVersions(ctx context.Context, bucketName string, objectsCh <-chan ObjectVersion, opts RemoveObjectsOptions) <-chan RemoveObjectError { errorCh := make(chan RemoveObjectError, 1) // Validate if bucket name is valid. @@ -181,12 +183,12 @@ func (c Client) RemoveObjects(ctx context.Context, bucketName string, objectsCh return errorCh } - go c.removeObjects(ctx, bucketName, objectsCh, errorCh, RemoveObjectsOptions{}) + go c.removeObjects(ctx, bucketName, objectsCh, errorCh, opts) return errorCh } // Generate and call MultiDelete S3 requests based on entries received from objectsCh -func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh <-chan string, errorCh chan<- RemoveObjectError, opts RemoveObjectsOptions) { +func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh <-chan ObjectVersion, errorCh chan<- RemoveObjectError, opts RemoveObjectsOptions) { maxEntries := 1000 finish := false urlValues := make(url.Values) @@ -201,7 +203,7 @@ func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh break } count := 0 - var batch []string + var batch []ObjectVersion // Try to gather 1000 entries for object := range objectsCh { @@ -246,7 +248,11 @@ func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh } if err != nil { for _, b := range batch { - errorCh <- RemoveObjectError{ObjectName: b, Err: err} + errorCh <- RemoveObjectError{ + ObjectName: b.Key, + VersionID: b.VersionID, + Err: err, + } } continue } @@ -263,31 +269,20 @@ type RemoveObjectsOptions struct { GovernanceBypass bool } -// RemoveObjectsWithOptions removes multiple objects from a bucket. +// RemoveObjects removes multiple objects from a bucket. // The list of objects to remove are received from objectsCh. // Remove failures are sent back via error channel. -func (c Client) RemoveObjectsWithOptions(ctx context.Context, bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError { - errorCh := make(chan RemoveObjectError, 1) - - // Validate if bucket name is valid. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - defer close(errorCh) - errorCh <- RemoveObjectError{ - Err: err, - } - return errorCh - } - // Validate objects channel to be properly allocated. - if objectsCh == nil { - defer close(errorCh) - errorCh <- RemoveObjectError{ - Err: errInvalidArgument("Objects channel cannot be nil"), +func (c Client) RemoveObjects(ctx context.Context, bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError { + objectsVersionsCh := make(chan ObjectVersion) + go func() { + defer close(objectsVersionsCh) + for obj := range objectsCh { + objectsVersionsCh <- ObjectVersion{ + Key: obj, + } } - return errorCh - } - - go c.removeObjects(ctx, bucketName, objectsCh, errorCh, opts) - return errorCh + }() + return c.RemoveObjectsWithVersions(ctx, bucketName, objectsVersionsCh, opts) } // RemoveIncompleteUpload aborts an partially uploaded object. diff --git a/api-s3-datatypes.go b/api-s3-datatypes.go index a6b125522..ac3445745 100644 --- a/api-s3-datatypes.go +++ b/api-s3-datatypes.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,9 @@ package minio import ( "encoding/xml" + "errors" + "io" + "reflect" "time" ) @@ -71,6 +74,103 @@ type ListBucketV2Result struct { StartAfter string } +// Version is an element in the list object versions response +type Version struct { + ETag string + IsLatest bool + Key string + LastModified time.Time + Owner Owner + Size int64 + StorageClass string + VersionID string `xml:"VersionId"` + + isDeleteMarker bool +} + +// ListVersionsResult is an element in the list object versions response +type ListVersionsResult struct { + Versions []Version + + CommonPrefixes []CommonPrefix + Name string + Prefix string + Delimiter string + MaxKeys int64 + EncodingType string + IsTruncated bool + KeyMarker string + VersionIDMarker string + NextKeyMarker string + NextVersionIDMarker string +} + +// UnmarshalXML is a custom unmarshal code for the response of ListObjectVersions, the custom +// code will unmarshal and tags and save them in Versions field to +// preserve the lexical order of the listing. +func (l *ListVersionsResult) UnmarshalXML(d *xml.Decoder, start xml.StartElement) (err error) { + for { + // Read tokens from the XML document in a stream. + t, err := d.Token() + if err != nil { + if err == io.EOF { + break + } + return err + } + + switch se := t.(type) { + case xml.StartElement: + tagName := se.Name.Local + switch tagName { + case "Name", "Prefix", + "Delimiter", "EncodingType", + "KeyMarker", "VersionIdMarker", + "NextKeyMarker", "NextVersionIdMarker": + var s string + if err = d.DecodeElement(&s, &se); err != nil { + return err + } + v := reflect.ValueOf(l).Elem().FieldByName(tagName) + if v.IsValid() { + v.SetString(s) + } + case "IsTruncated": // bool + var b bool + if err = d.DecodeElement(&b, &se); err != nil { + return err + } + l.IsTruncated = b + case "MaxKeys": // int64 + var i int64 + if err = d.DecodeElement(&i, &se); err != nil { + return err + } + l.MaxKeys = i + case "CommonPrefixes": + var cp CommonPrefix + if err = d.DecodeElement(&cp, &se); err != nil { + return err + } + l.CommonPrefixes = append(l.CommonPrefixes, cp) + case "DeleteMarker", "Version": + var v Version + if err = d.DecodeElement(&v, &se); err != nil { + return err + } + if tagName == "DeleteMarker" { + v.isDeleteMarker = true + } + l.Versions = append(l.Versions, v) + default: + return errors.New("unrecognized option:" + tagName) + } + + } + } + return nil +} + // ListBucketResult container for listObjects response. type ListBucketResult struct { // A response can contain CommonPrefixes only if you have diff --git a/api-stat.go b/api-stat.go index 51f9ae9f3..0cdac7e71 100644 --- a/api-stat.go +++ b/api-stat.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ package minio import ( "context" "net/http" + "net/url" "github.com/minio/minio-go/v7/pkg/s3utils" ) @@ -78,10 +79,16 @@ func (c Client) statObject(ctx context.Context, bucketName, objectName string, o return ObjectInfo{}, err } + urlValues := make(url.Values) + if opts.VersionID != "" { + urlValues.Set("versionId", opts.VersionID) + } + // Execute HEAD on objectName. resp, err := c.executeMethod(ctx, "HEAD", requestMetadata{ bucketName: bucketName, objectName: objectName, + queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, customHeader: opts.Header(), }) diff --git a/core.go b/core.go index 605e7ac5f..a7f6c7281 100644 --- a/core.go +++ b/core.go @@ -70,7 +70,7 @@ func (c Core) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBuck } // PutObject - Upload object. Uploads using single PUT call. -func (c Core) PutObject(ctx context.Context, bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, opts PutObjectOptions) (ObjectInfo, error) { +func (c Core) PutObject(ctx context.Context, bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, opts PutObjectOptions) (UploadInfo, error) { return c.putObjectDo(ctx, bucket, object, data, md5Base64, sha256Hex, size, opts) } diff --git a/core_test.go b/core_test.go index 5b0509e06..61a01540d 100644 --- a/core_test.go +++ b/core_test.go @@ -103,15 +103,19 @@ func TestGetObjectCore(t *testing.T) { // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.Client.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), PutObjectOptions{ + _, err = c.Client.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), PutObjectOptions{ ContentType: "binary/octet-stream", }) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } - if n != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + st, err := c.Client.StatObject(context.Background(), bucketName, objectName, StatObjectOptions{}) + if err != nil { + t.Fatal("Stat error:", err, bucketName, objectName) + } + if st.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size) } offset := int64(2048) @@ -119,7 +123,7 @@ func TestGetObjectCore(t *testing.T) { // read directly buf1 := make([]byte, 512) buf2 := make([]byte, 512) - buf3 := make([]byte, n) + buf3 := make([]byte, st.Size) buf4 := make([]byte, 1) opts := GetObjectOptions{} @@ -245,7 +249,7 @@ func TestGetObjectCore(t *testing.T) { t.Fatalf("Error: Content Length in response header %v, not equal to set content length %v\n", contentLengthValue, contentLength) } - err = c.RemoveObject(context.Background(), bucketName, objectName) + err = c.RemoveObject(context.Background(), bucketName, objectName, RemoveObjectOptions{}) if err != nil { t.Fatal("Error: ", err) } @@ -296,24 +300,20 @@ func TestGetObjectContentEncoding(t *testing.T) { // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.Client.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), PutObjectOptions{ + _, err = c.Client.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), PutObjectOptions{ ContentEncoding: "gzip", }) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } - if n != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - rwc, objInfo, _, err := c.GetObject(context.Background(), bucketName, objectName, GetObjectOptions{}) if err != nil { t.Fatalf("Error: %v", err) } rwc.Close() - if objInfo.Size <= 0 { - t.Fatalf("Unexpected size of the object %v, expected %v", objInfo.Size, n) + if objInfo.Size != int64(len(buf)) { + t.Fatalf("Unexpected size of the object %v, expected %v", objInfo.Size, len(buf)) } value, ok := objInfo.Metadata["Content-Encoding"] if !ok { @@ -323,7 +323,7 @@ func TestGetObjectContentEncoding(t *testing.T) { t.Fatalf("Unexpected content-encoding found, want gzip, got %v", value) } - err = c.RemoveObject(context.Background(), bucketName, objectName) + err = c.RemoveObject(context.Background(), bucketName, objectName, RemoveObjectOptions{}) if err != nil { t.Fatal("Error: ", err) } @@ -441,27 +441,32 @@ func TestCoreCopyObject(t *testing.T) { "Content-Type": "binary/octet-stream", }, } - objInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", putopts) + uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", putopts) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } - if objInfo.Size != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size) + st, err := c.StatObject(context.Background(), bucketName, objectName, StatObjectOptions{}) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if st.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size) } destBucketName := bucketName destObjectName := objectName + "-dest" - cobjInfo, err := c.CopyObject(context.Background(), bucketName, objectName, destBucketName, destObjectName, map[string]string{ + cuploadInfo, err := c.CopyObject(context.Background(), bucketName, objectName, destBucketName, destObjectName, map[string]string{ "X-Amz-Metadata-Directive": "REPLACE", "Content-Type": "application/javascript", }) if err != nil { t.Fatal("Error:", err, bucketName, objectName, destBucketName, destObjectName) } - if cobjInfo.ETag != objInfo.ETag { - t.Fatalf("Error: expected etag to be same as source object %s, but found different etag %s", objInfo.ETag, cobjInfo.ETag) + if cuploadInfo.ETag != uploadInfo.ETag { + t.Fatalf("Error: expected etag to be same as source object %s, but found different etag %s", uploadInfo.ETag, cuploadInfo.ETag) } // Attempt to read from destBucketName and object name. @@ -470,7 +475,7 @@ func TestCoreCopyObject(t *testing.T) { t.Fatal("Error:", err, bucketName, objectName) } - st, err := r.Stat() + st, err = r.Stat() if err != nil { t.Fatal("Error:", err, bucketName, objectName) } @@ -484,8 +489,8 @@ func TestCoreCopyObject(t *testing.T) { t.Fatalf("Error: Content types don't match, expected: application/javascript, found: %+v\n", st.ContentType) } - if st.ETag != objInfo.ETag { - t.Fatalf("Error: expected etag to be same as source object %s, but found different etag :%s", objInfo.ETag, st.ETag) + if st.ETag != uploadInfo.ETag { + t.Fatalf("Error: expected etag to be same as source object %s, but found different etag :%s", uploadInfo.ETag, st.ETag) } if err := r.Close(); err != nil { @@ -496,12 +501,12 @@ func TestCoreCopyObject(t *testing.T) { t.Fatal("Error: object is already closed, should return error") } - err = c.RemoveObject(context.Background(), bucketName, objectName) + err = c.RemoveObject(context.Background(), bucketName, objectName, RemoveObjectOptions{}) if err != nil { t.Fatal("Error: ", err) } - err = c.RemoveObject(context.Background(), destBucketName, destObjectName) + err = c.RemoveObject(context.Background(), destBucketName, destObjectName, RemoveObjectOptions{}) if err != nil { t.Fatal("Error: ", err) } @@ -559,13 +564,18 @@ func TestCoreCopyObjectPart(t *testing.T) { } // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - objInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", putopts) + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", putopts) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } - if objInfo.Size != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size) + st, err := c.StatObject(context.Background(), bucketName, objectName, StatObjectOptions{}) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if st.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size) } destBucketName := bucketName @@ -605,7 +615,7 @@ func TestCoreCopyObjectPart(t *testing.T) { } // Stat the object and check its length matches - objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, StatObjectOptions{}) + objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, StatObjectOptions{}) if err != nil { t.Fatal("Error:", err, destBucketName, destObjectName) } @@ -647,11 +657,11 @@ func TestCoreCopyObjectPart(t *testing.T) { t.Fatal("Got unexpected data in last byte of copied object!") } - if err := c.RemoveObject(context.Background(), destBucketName, destObjectName); err != nil { + if err := c.RemoveObject(context.Background(), destBucketName, destObjectName, RemoveObjectOptions{}); err != nil { t.Fatal("Error: ", err) } - if err := c.RemoveObject(context.Background(), bucketName, objectName); err != nil { + if err := c.RemoveObject(context.Background(), bucketName, objectName, RemoveObjectOptions{}); err != nil { t.Fatal("Error: ", err) } @@ -709,20 +719,16 @@ func TestCorePutObject(t *testing.T) { putopts := PutObjectOptions{ UserMetadata: metadata, } - objInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "1B2M2Y8AsgTpgAmY7PhCfg==", "", putopts) + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "1B2M2Y8AsgTpgAmY7PhCfg==", "", putopts) if err == nil { t.Fatal("Error expected: error, got: nil(success)") } - objInfo, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", putopts) + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", putopts) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } - if objInfo.Size != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size) - } - // Read the data back r, err := c.Client.GetObject(context.Background(), bucketName, objectName, GetObjectOptions{}) if err != nil { @@ -751,7 +757,7 @@ func TestCorePutObject(t *testing.T) { t.Fatal("Error: object is already closed, should return error") } - err = c.RemoveObject(context.Background(), bucketName, objectName) + err = c.RemoveObject(context.Background(), bucketName, objectName, RemoveObjectOptions{}) if err != nil { t.Fatal("Error: ", err) } @@ -808,7 +814,7 @@ func TestCoreGetObjectMetadata(t *testing.T) { log.Fatalln("Expected metadata to be available but wasn't") } - err = core.RemoveObject(context.Background(), bucketName, "my-objectname") + err = core.RemoveObject(context.Background(), bucketName, "my-objectname", RemoveObjectOptions{}) if err != nil { t.Fatal("Error: ", err) } diff --git a/docs/API.md b/docs/API.md index 493205404..61067f23b 100644 --- a/docs/API.md +++ b/docs/API.md @@ -64,8 +64,6 @@ func main() { | [`GetBucketTagging`](#GetBucketTagging) | [`ComposeObject`](#ComposeObject) | | | [`GetObjectLockConfig`](#GetObjectLockConfig) | | | [`DeleteBucketTagging`](#DeleteBucketTagging) | [`NewSourceInfo`](#NewSourceInfo) | | | [`EnableVersioning`](#EnableVersioning) | | | | [`NewDestinationInfo`](#NewDestinationInfo) | | | [`DisableVersioning`](#DisableVersioning) | | -| | [`RemoveObjectsWithOptions`](#RemoveObjectsWithOptions) | | | [`GetBucketVersioning`](#GetBucketVersioning) | | -| | [`RemoveObjectWithOptions`](#RemoveObjectWithOptions) | | | [`SetBucketEncryption`](#SetBucketEncryption) | | | | [`PutObjectRetention`](#PutObjectRetention) | | | [`GetBucketEncryption`](#GetBucketEncryption) | | | | [`GetObjectRetention`](#GetObjectRetention) | | | [`DeleteBucketEncryption`](#DeleteBucketEncryption) | | | | [`PutObjectLegalHold`](#PutObjectLegalHold) | | | | | @@ -581,12 +579,11 @@ if err != nil { ``` -### PutObject(ctx context.Context, bucketName, objectName string, reader io.Reader, objectSize int64,opts PutObjectOptions) (n int, err error) +### PutObject(ctx context.Context, bucketName, objectName string, reader io.Reader, objectSize int64,opts PutObjectOptions) (info UploadInfo, err error) Uploads objects that are less than 128MiB in a single PUT operation. For objects that are greater than 128MiB in size, PutObject seamlessly uploads the object as parts of 128MiB or more depending on the actual file size. The max upload size for an object is 5TB. __Parameters__ - |Param |Type |Description | |:---|:---| :---| |`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| @@ -619,6 +616,14 @@ __minio.PutObjectOptions__ | `opts.ReplicationStatus` | _string_ | Specify replication status of object. This option is intended for internal use by MinIO server to extend the replication API implementation by AWS. This option should not be set unless the application is aware of intended use. | +__minio.UploadInfo__ + +| Field | Type | Description | +|:--------------------|:---------|:-------------------------------------------| +| `info.ETag` | _string_ | The ETag of the new object | +| `info.VersionID` | _string_ | The version identifyer of the new object | + + __Example__ @@ -636,19 +641,19 @@ if err != nil { return } -n, err := minioClient.PutObject(context.Background(), "mybucket", "myobject", file, fileStat.Size(), minio.PutObjectOptions{ContentType:"application/octet-stream"}) +uploadInfo, err := minioClient.PutObject(context.Background(), "mybucket", "myobject", file, fileStat.Size(), minio.PutObjectOptions{ContentType:"application/octet-stream"}) if err != nil { fmt.Println(err) return } -fmt.Println("Successfully uploaded bytes: ", n) +fmt.Println("Successfully uploaded bytes: ", uploadInfo) ``` API methods PutObjectWithSize, PutObjectWithMetadata, PutObjectStreaming, and PutObjectWithProgress available in minio-go SDK release v3.0.3 are replaced by the new PutObject call variant that accepts a pointer to PutObjectOptions struct. -### CopyObject(ctx context.Context, dst DestinationInfo, src SourceInfo) error +### CopyObject(ctx context.Context, dst DestinationInfo, src SourceInfo) (UploadInfo, error) Create or replace an object through server-side copying of an existing object. It supports conditional copying, copying a part of an object and server-side encryption of destination and decryption of source. See the `SourceInfo` and `DestinationInfo` types for further details. To copy multiple source objects into a single destination object see the `ComposeObject` API. @@ -663,6 +668,16 @@ __Parameters__ |`src` | _minio.SourceInfo_ |Argument describing the source object | +__minio.UploadInfo__ + +| Field | Type | Description | +|:--------------------|:---------|:-------------------------------------------| +| `info.ETag` | _string_ | The ETag of the new object | +| `info.VersionID` | _string_ | The version identifyer of the new object | + + + + __Example__ @@ -679,11 +694,13 @@ if err != nil { } // Copy object call -err = minioClient.CopyObject(context.Background(), dst, src) +uploadInfo, err := minioClient.CopyObject(context.Background(), dst, src) if err != nil { fmt.Println(err) return } + +fmt.Println("Successfully copied object:", uploadInfo) ``` ```go @@ -717,15 +734,18 @@ if err != nil { } // Copy object call -err = minioClient.CopyObject(context.Background(), dst, src) +_, err = minioClient.CopyObject(context.Background(), dst, src) if err != nil { fmt.Println(err) return } + +fmt.Println("Successfully copied object:", uploadInfo) + ``` -### ComposeObject(ctx context.Context, dst minio.DestinationInfo, srcs []minio.SourceInfo) error +### ComposeObject(ctx context.Context, dst minio.DestinationInfo, srcs []minio.SourceInfo) (UploadInfo, error) Create an object by concatenating a list of source objects using server-side copying. __Parameters__ @@ -738,6 +758,15 @@ __Parameters__ |`srcs` | _[]minio.SourceInfo_ |Slice of struct with info about source objects to be concatenated in order. | +__minio.UploadInfo__ + +| Field | Type | Description | +|:--------------------|:---------|:-------------------------------------------| +| `info.ETag` | _string_ | The ETag of the new object | +| `info.VersionID` | _string_ | The version identifyer of the new object | + + + __Example__ @@ -771,13 +800,13 @@ if err != nil { } // Compose object call by concatenating multiple source files. -err = minioClient.ComposeObject(context.Background(), dst, srcs) +uploadInfo, err := minioClient.ComposeObject(context.Background(), dst, srcs) if err != nil { fmt.Println(err) return } -fmt.Println("Composed object successfully.") +fmt.Println("Composed object successfully:", uploadInfo) ``` @@ -826,7 +855,7 @@ if err != nil { } // Copy object call -err = minioClient.CopyObject(dst, src) +_, err = minioClient.CopyObject(dst, src) if err != nil { fmt.Println(err) return @@ -858,7 +887,7 @@ if err != nil { } // Copy object call -err = minioClient.CopyObject(dst, src) +_, err = minioClient.CopyObject(dst, src) if err != nil { fmt.Println(err) return @@ -877,15 +906,15 @@ if err != nil { } // Copy object call -err = minioClient.CopyObject(dst, src) +_, err = minioClient.CopyObject(dst, src) if err != nil { fmt.Println(err) return } ``` - -### NewDestinationInfoWithOptions(bucket, object string, destOpts DestInfoOptions) (DestinationInfo, error) + +### NewDestinationInfo(bucket, object string, destOpts DestInfoOptions) (DestinationInfo, error) Construct a `DestinationInfo` object that can be used as the destination object for server-side copying operations like `CopyObject` and `ComposeObject`. __Parameters__ @@ -917,7 +946,7 @@ tags := map[string]string{ "Tag1": "Value1", "Tag2": "Value2", } -dst, err := minio.NewDestinationInfoWithOptions("bucket", "object", minio.DestInfoOptions{ +dst, err := minio.NewDestinationInfo("bucket", "object", minio.DestInfoOptions{ UserTags: tags, ReplaceTags: true, }) if err != nil { @@ -926,7 +955,7 @@ if err != nil { } // Copy object call -err = minioClient.CopyObject(context.Background(), dst, src) +_, err = minioClient.CopyObject(context.Background(), dst, src) if err != nil { fmt.Println(err) return @@ -942,7 +971,7 @@ tags := map[string]string{ "Tag1": "Value1", "Tag2": "Value2", } -dst, err := minio.NewDestinationInfoWithOptions("bucket", "object", minio.DestInfoOptions{ +dst, err := minio.NewDestinationInfo("bucket", "object", minio.DestInfoOptions{ Encryption: sseDst, UserTags: tags, ReplaceTags: true, }) if err != nil { @@ -951,7 +980,7 @@ if err != nil { } // Copy object call -err = minioClient.CopyObject(context.Background(), dst, src) +_, err = minioClient.CopyObject(context.Background(), dst, src) if err != nil { fmt.Println(err) return @@ -959,7 +988,7 @@ if err != nil { ``` -### FPutObject(ctx context.Context, bucketName, objectName, filePath, opts PutObjectOptions) (length int64, err error) +### FPutObject(ctx context.Context, bucketName, objectName, filePath, opts PutObjectOptions) (info UploadInfo, err error) Uploads contents from a file to objectName. FPutObject uploads objects that are less than 128MiB in a single PUT operation. For objects that are greater than the 128MiB in size, FPutObject seamlessly uploads the object in chunks of 128MiB or more depending on the actual file size. The max upload size for an object is 5TB. @@ -976,18 +1005,26 @@ __Parameters__ |`opts` | _minio.PutObjectOptions_ |Pointer to struct that allows user to set optional custom metadata, content-type, content-encoding, content-disposition, content-language and cache-control headers, pass encryption module for encrypting objects, and optionally configure number of threads for multipart put operation. | +__minio.UploadInfo__ + +| Field | Type | Description | +|:--------------------|:---------|:-------------------------------------------| +| `info.ETag` | _string_ | The ETag of the new object | +| `info.VersionID` | _string_ | The version identifyer of the new object | + + __Example__ ```go -n, err := minioClient.FPutObject(context.Background(), "my-bucketname", "my-objectname", "my-filename.csv", minio.PutObjectOptions{ +uploadInfo, err := minioClient.FPutObject(context.Background(), "my-bucketname", "my-objectname", "my-filename.csv", minio.PutObjectOptions{ ContentType: "application/csv", }); if err != nil { fmt.Println(err) return } -fmt.Println("Successfully uploaded bytes: ", n) +fmt.Println("Successfully uploaded object: ", uploadInfo) ``` @@ -1035,70 +1072,8 @@ fmt.Println(objInfo) ``` -### RemoveObject(ctx context.Context, bucketName, objectName string) error -Removes an object. - -__Parameters__ - - -|Param |Type |Description | -|:---|:---| :---| -|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| -|`bucketName` | _string_ |Name of the bucket | -|`objectName` | _string_ |Name of the object | - - -```go -err = minioClient.RemoveObject(context.Background(), "mybucket", "myobject") -if err != nil { - fmt.Println(err) - return -} -``` - - -### RemoveObjects(ctx context.Context, bucketName string, objectsCh chan string) (errorCh <-chan RemoveObjectError) -Removes a list of objects obtained from an input channel. The call sends a delete request to the server up to 1000 objects at a time. The errors observed are sent over the error channel. - -__Parameters__ - -|Param |Type |Description | -|:---|:---| :---| -|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| -|`bucketName` | _string_ |Name of the bucket | -|`objectsCh` | _chan string_ | Channel of objects to be removed | - - -__Return Values__ - -|Param |Type |Description | -|:---|:---| :---| -|`errorCh` | _<-chan minio.RemoveObjectError_ | Receive-only channel of errors observed during deletion. | - - -```go -objectsCh := make(chan string) - -// Send object names that are needed to be removed to objectsCh -go func() { - defer close(objectsCh) - // List all objects from a bucket-name with a matching prefix. - for object := range minioClient.ListObjects(context.Background(), "my-bucketname", "my-prefixname", true, nil) { - if object.Err != nil { - log.Fatalln(object.Err) - } - objectsCh <- object.Key - } -}() - -for rErr := range minioClient.RemoveObjects(context.Background(), "mybucket", objectsCh) { - fmt.Println("Error detected during deletion: ", rErr) -} -``` - - -### RemoveObjectWithOptions(ctx context.Context, bucketName, objectName string, opts minio.RemoveObjectOptions) error -Removes an object with more specified options +### RemoveObject(ctx context.Context, bucketName, objectName string, opts minio.RemoveObjectOptions) error +Removes an object with some specified options __Parameters__ @@ -1123,7 +1098,7 @@ opts := minio.RemoveObjectOptions { GovernanceBypass: true, VersionID: "myversionid", } -err = minioClient.RemoveObjectWithOptions(context.Background(), "mybucket", "myobject", opts) +err = minioClient.RemoveObject(context.Background(), "mybucket", "myobject", opts) if err != nil { fmt.Println(err) return @@ -1143,9 +1118,9 @@ __Parameters__ |`objectName` | _string_ |Name of the object | |`opts` |_minio.PutObjectRetentionOptions_ |Allows user to set options like retention mode, expiry date and version id | - -### RemoveObjectsWithOptions(ctx context.Context, bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError -*Identical to RemoveObjects operation, but accepts opts for bypassing Governance mode.* + +### RemoveObjects(ctx context.Context, bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError +Removes a list of objects obtained from an input channel. The call sends a delete request to the server up to 1000 objects at a time. The errors observed are sent over the error channel. Parameters @@ -1187,7 +1162,7 @@ opts := minio.RemoveObjectsOptions{ GovernanceBypass: true, } -for rErr := range minioClient.RemoveObjectsWithOptions(context.Background(), "my-bucketname", objectsCh, opts) { +for rErr := range minioClient.RemoveObjects(context.Background(), "my-bucketname", objectsCh, opts) { fmt.Println("Error detected during deletion: ", rErr) } ``` diff --git a/examples/s3/composeobject.go b/examples/s3/composeobject.go index c628c2f29..33ee0396c 100644 --- a/examples/s3/composeobject.go +++ b/examples/s3/composeobject.go @@ -66,15 +66,15 @@ func main() { encKey, _ := encrypt.NewSSEC([]byte{8, 9, 0}) // Create destination info - dst, err := minio.NewDestinationInfo("bucket", "object", encKey, nil) + dst, err := minio.NewDestinationInfo("bucket", "object", minio.DestInfoOptions{Encryption: encKey}) if err != nil { log.Fatalln(err) } - err = s3Client.ComposeObject(context.Background(), dst, srcs) + uploadInfo, err := s3Client.ComposeObject(context.Background(), dst, srcs) if err != nil { log.Fatalln(err) } - log.Println("Composed object successfully.") + log.Println("Composed object successfully:", uploadInfo) } diff --git a/examples/s3/copyobject-with-new-tags.go b/examples/s3/copyobject-with-new-tags.go index b0496d1cb..744fce587 100644 --- a/examples/s3/copyobject-with-new-tags.go +++ b/examples/s3/copyobject-with-new-tags.go @@ -67,14 +67,15 @@ func main() { } // Destination object - dst, err := minio.NewDestinationInfoWithOptions("my-bucketname", "my-objectname", minio.DestInfoOptions{ + dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", minio.DestInfoOptions{ UserTags: tags, ReplaceTags: true, }) if err != nil { log.Fatalln(err) } + // Initiate copy object. - err = s3Client.CopyObject(context.Background(), dst, src) + _, err = s3Client.CopyObject(context.Background(), dst, src) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/copyobject.go b/examples/s3/copyobject.go index 390bb5d99..a235a910d 100644 --- a/examples/s3/copyobject.go +++ b/examples/s3/copyobject.go @@ -62,13 +62,13 @@ func main() { // src.SetMatchETagExceptCond("31624deb84149d2f8ef9c385918b653a") // Destination object - dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil) + dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", minio.DestInfoOptions{}) if err != nil { log.Fatalln(err) } // Initiate copy object. - err = s3Client.CopyObject(context.Background(), dst, src) + _, err = s3Client.CopyObject(context.Background(), dst, src) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/fputencrypted-object.go b/examples/s3/fputencrypted-object.go index 1d03beca7..834f79dcc 100644 --- a/examples/s3/fputencrypted-object.go +++ b/examples/s3/fputencrypted-object.go @@ -50,10 +50,10 @@ func main() { encryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketname+objectName)) // Encrypt file content and upload to the server - n, err := s3Client.FPutObject(context.Background(), bucketname, objectName, filePath, minio.PutObjectOptions{ServerSideEncryption: encryption}) + uploadedInfo, err := s3Client.FPutObject(context.Background(), bucketname, objectName, filePath, minio.PutObjectOptions{ServerSideEncryption: encryption}) if err != nil { log.Fatalln(err) } - log.Println("Uploaded", "my-objectname", " of size: ", n, "Successfully.") + log.Println("Uploaded", "my-objectname:", uploadedInfo) } diff --git a/examples/s3/removeobjectoptions.go b/examples/s3/listobjectversions.go similarity index 57% rename from examples/s3/removeobjectoptions.go rename to examples/s3/listobjectversions.go index edc408468..8ce3d1806 100644 --- a/examples/s3/removeobjectoptions.go +++ b/examples/s3/listobjectversions.go @@ -2,7 +2,7 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2019 MinIO, Inc. + * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,30 +21,39 @@ package main import ( "context" - "log" + "fmt" "github.com/minio/minio-go/v7" ) func main() { - // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and - // my-testfile are dummy values, please replace them with original values. + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-prefixname + // are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) if err != nil { - log.Fatalln(err) + fmt.Println(err) + return } - opts := minio.RemoveObjectOptions{ - GovernanceBypass: true, - } - err = s3Client.RemoveObjectWithOptions(context.Background(), "my-bucket", "my-object", opts) - if err != nil { - log.Fatalln(err) + + // Create a done channel to control 'ListObjects' go routine. + doneCh := make(chan struct{}) + + // Indicate to our routine to exit cleanly upon return. + defer close(doneCh) + + // List all objects from a bucket-name with a matching prefix. + for objectVersion := range s3Client.ListObjectVersions(context.Background(), "my-bucketname", "my-prefixname", true, doneCh) { + if objectVersion.Err != nil { + fmt.Println(objectVersion.Err) + return + } + fmt.Println(objectVersion) } - log.Println("Remove object successful") + return } diff --git a/examples/s3/removeobject.go b/examples/s3/removeobject.go index 547b5aa4e..9a42ba425 100644 --- a/examples/s3/removeobject.go +++ b/examples/s3/removeobject.go @@ -39,9 +39,15 @@ func main() { if err != nil { log.Fatalln(err) } - err = s3Client.RemoveObject(context.Background(), "my-bucketname", "my-objectname") + + opts := minio.RemoveObjectOptions{ + GovernanceBypass: true, + } + + err = s3Client.RemoveObject(context.Background(), "my-bucketname", "my-objectname", opts) if err != nil { log.Fatalln(err) } + log.Println("Success") } diff --git a/functional_tests.go b/functional_tests.go index 27ab851c4..e259146c0 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -34,6 +34,7 @@ import ( "path/filepath" "reflect" "runtime" + "sort" "strconv" "strings" "time" @@ -148,12 +149,12 @@ func cleanupBucket(bucketName string, c *minio.Client) error { // Exit cleanly upon return. defer close(doneCh) // Iterate over all objects in the bucket via listObjectsV2 and delete - for objCh := range c.ListObjectsV2(context.Background(), bucketName, "", true, doneCh) { + for objCh := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{Recursive: true}) { if objCh.Err != nil { return objCh.Err } if objCh.Key != "" { - err := c.RemoveObject(context.Background(), bucketName, objCh.Key) + err := c.RemoveObject(context.Background(), bucketName, objCh.Key, minio.RemoveObjectOptions{}) if err != nil { return err } @@ -178,6 +179,40 @@ func cleanupBucket(bucketName string, c *minio.Client) error { return err } +func cleanupVersionedBucket(bucketName string, c *minio.Client) error { + doneCh := make(chan struct{}) + defer close(doneCh) + for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) { + if obj.Err != nil { + return obj.Err + } + if obj.Key != "" { + err := c.RemoveObject(context.Background(), bucketName, obj.Key, + minio.RemoveObjectOptions{VersionID: obj.VersionID, GovernanceBypass: true}) + if err != nil { + return err + } + } + } + for objPartInfo := range c.ListIncompleteUploads(context.Background(), bucketName, "", true, doneCh) { + if objPartInfo.Err != nil { + return objPartInfo.Err + } + if objPartInfo.Key != "" { + err := c.RemoveIncompleteUpload(context.Background(), bucketName, objPartInfo.Key) + if err != nil { + return err + } + } + } + // objects are already deleted, clear the buckets now + err := c.RemoveBucket(context.Background(), bucketName) + if err != nil { + return err + } + return err +} + func isErrNotImplemented(err error) bool { return minio.ToErrorResponse(err).Code == "NotImplemented" } @@ -525,66 +560,1046 @@ func testPutObjectReadAt() { args["bucketName"] = bucketName // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) + if err != nil { + logError(testName, function, args, startTime, "", "Make bucket failed", err) + return + } + + bufSize := dataFileMap["datafile-129-MB"] + var reader = getDataReader("datafile-129-MB") + defer reader.Close() + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + // Object content type + objectContentType := "binary/octet-stream" + args["objectContentType"] = objectContentType + + _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: objectContentType}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + // Read the data back + r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "Get Object failed", err) + return + } + + st, err := r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat Object failed", err) + return + } + if st.Size != int64(bufSize) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Number of bytes in stat does not match, expected %d got %d", bufSize, st.Size), err) + return + } + if st.ContentType != objectContentType && st.ContentType != "application/octet-stream" { + logError(testName, function, args, startTime, "", "Content types don't match", err) + return + } + if err := r.Close(); err != nil { + logError(testName, function, args, startTime, "", "Object Close failed", err) + return + } + if err := r.Close(); err == nil { + logError(testName, function, args, startTime, "", "Object is already closed, didn't return error on Close", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +func testListObjectVersions() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "ListObjectVersions(bucketName, prefix, recursive)" + args := map[string]interface{}{ + "bucketName": "", + "prefix": "", + "recursive": "", + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) + if err != nil { + logError(testName, function, args, startTime, "", "Make bucket failed", err) + return + } + + err = c.EnableVersioning(context.Background(), bucketName) + if err != nil { + logError(testName, function, args, startTime, "", "Enable versioning failed", err) + return + } + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + bufSize := dataFileMap["datafile-10-kB"] + var reader = getDataReader("datafile-10-kB") + + _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + reader.Close() + + bufSize = dataFileMap["datafile-1-b"] + reader = getDataReader("datafile-1-b") + _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + reader.Close() + + err = c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "Unexpected object deletion", err) + return + } + + var deleteMarkers, versions int + + objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) + for info := range objectsInfo { + if info.Err != nil { + logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) + return + } + if info.Key != objectName { + logError(testName, function, args, startTime, "", "Unexpected object name in listing objects", nil) + return + } + if info.VersionID == "" { + logError(testName, function, args, startTime, "", "Unexpected version id in listing objects", nil) + return + } + if info.IsDeleteMarker { + deleteMarkers++ + if !info.IsLatest { + logError(testName, function, args, startTime, "", "Unexpected IsLatest field in listing objects", nil) + return + } + } else { + versions++ + } + } + + if deleteMarkers != 1 { + logError(testName, function, args, startTime, "", "Unexpected number of DeleteMarker elements in listing objects", nil) + return + } + + if versions != 2 { + logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil) + return + } + + // Delete all objects and their versions as long as the bucket itself + if err = cleanupVersionedBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +func testStatObjectWithVersioning() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "StatObject" + args := map[string]interface{}{} + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) + if err != nil { + logError(testName, function, args, startTime, "", "Make bucket failed", err) + return + } + + err = c.EnableVersioning(context.Background(), bucketName) + if err != nil { + logError(testName, function, args, startTime, "", "Enable versioning failed", err) + return + } + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + bufSize := dataFileMap["datafile-10-kB"] + var reader = getDataReader("datafile-10-kB") + + _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + reader.Close() + + bufSize = dataFileMap["datafile-1-b"] + reader = getDataReader("datafile-1-b") + _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + reader.Close() + + objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) + + var results []minio.ObjectInfo + for info := range objectsInfo { + if info.Err != nil { + logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) + return + } + results = append(results, info) + } + + if len(results) != 2 { + logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil) + return + } + + for i := 0; i < len(results); i++ { + opts := minio.StatObjectOptions{minio.GetObjectOptions{VersionID: results[i].VersionID}} + statInfo, err := c.StatObject(context.Background(), bucketName, objectName, opts) + if err != nil { + logError(testName, function, args, startTime, "", "error during HEAD object", err) + return + } + if statInfo.ETag != results[i].ETag { + logError(testName, function, args, startTime, "", "error during HEAD object, unexpected ETag", err) + return + } + if statInfo.LastModified.Unix() != results[i].LastModified.Unix() { + logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Last-Modified", err) + return + } + if statInfo.Size != results[i].Size { + logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Content-Length", err) + return + } + } + + // Delete all objects and their versions as long as the bucket itself + if err = cleanupVersionedBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +func testGetObjectWithVersioning() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "GetObject()" + args := map[string]interface{}{} + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) + if err != nil { + logError(testName, function, args, startTime, "", "Make bucket failed", err) + return + } + + err = c.EnableVersioning(context.Background(), bucketName) + if err != nil { + logError(testName, function, args, startTime, "", "Enable versioning failed", err) + return + } + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + // Save the contents of datafiles to check with GetObject() reader output later + var buffers [][]byte + var testFiles = []string{"datafile-1-b", "datafile-10-kB"} + + for _, testFile := range testFiles { + r := getDataReader(testFile) + buf, err := ioutil.ReadAll(r) + if err != nil { + logError(testName, function, args, startTime, "", "unexpected failure", err) + return + } + r.Close() + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + buffers = append(buffers, buf) + } + + objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) + + var results []minio.ObjectInfo + for info := range objectsInfo { + if info.Err != nil { + logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) + return + } + results = append(results, info) + } + + if len(results) != 2 { + logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil) + return + } + + sort.SliceStable(results, func(i, j int) bool { + return results[i].Size < results[j].Size + }) + + sort.SliceStable(buffers, func(i, j int) bool { + return len(buffers[i]) < len(buffers[j]) + }) + + for i := 0; i < len(results); i++ { + opts := minio.GetObjectOptions{VersionID: results[i].VersionID} + reader, err := c.GetObject(context.Background(), bucketName, objectName, opts) + if err != nil { + logError(testName, function, args, startTime, "", "error during GET object", err) + return + } + statInfo, err := reader.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "error during calling reader.Stat()", err) + return + } + if statInfo.ETag != results[i].ETag { + logError(testName, function, args, startTime, "", "error during HEAD object, unexpected ETag", err) + return + } + if statInfo.LastModified.Unix() != results[i].LastModified.Unix() { + logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Last-Modified", err) + return + } + if statInfo.Size != results[i].Size { + logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Content-Length", err) + return + } + + tmpBuffer := bytes.NewBuffer([]byte{}) + _, err = io.Copy(tmpBuffer, reader) + if err != nil { + logError(testName, function, args, startTime, "", "unexpected io.Copy()", err) + return + } + + if !bytes.Equal(tmpBuffer.Bytes(), buffers[i]) { + fmt.Println(len(tmpBuffer.Bytes()), len(buffers[i])) + logError(testName, function, args, startTime, "", "unexpected content of GetObject()", err) + return + } + } + + // Delete all objects and their versions as long as the bucket itself + if err = cleanupVersionedBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +func testCopyObjectWithVersioning() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObject()" + args := map[string]interface{}{} + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) + if err != nil { + logError(testName, function, args, startTime, "", "Make bucket failed", err) + return + } + + err = c.EnableVersioning(context.Background(), bucketName) + if err != nil { + logError(testName, function, args, startTime, "", "Enable versioning failed", err) + return + } + + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + var testFiles = []string{"datafile-1-b", "datafile-10-kB"} + for _, testFile := range testFiles { + r := getDataReader(testFile) + buf, err := ioutil.ReadAll(r) + if err != nil { + logError(testName, function, args, startTime, "", "unexpected failure", err) + return + } + r.Close() + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + } + + objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) + var infos []minio.ObjectInfo + for info := range objectsInfo { + if info.Err != nil { + logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) + return + } + infos = append(infos, info) + } + + sort.Slice(infos, func(i, j int) bool { + return infos[i].Size < infos[j].Size + }) + + reader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{VersionID: infos[0].VersionID}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject of the oldest version content failed", err) + return + } + + oldestContent, err := ioutil.ReadAll(reader) + if err != nil { + logError(testName, function, args, startTime, "", "Reading the oldest object version failed", err) + return + } + + // Copy Source + src := minio.NewSourceInfo(bucketName, objectName, nil) + src.SetVersionID(infos[0].VersionID) + args["src"] = src + + dst, err := minio.NewDestinationInfo(bucketName, objectName+"-copy", minio.DestInfoOptions{}) + args["dst"] = dst + if err != nil { + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) + return + } + + // Perform the Copy + _, err = c.CopyObject(context.Background(), dst, src) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObject failed", err) + return + } + + // Destination object + readerCopy, err := c.GetObject(context.Background(), bucketName, objectName+"-copy", minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + defer readerCopy.Close() + + newestContent, err := ioutil.ReadAll(readerCopy) + if err != nil { + logError(testName, function, args, startTime, "", "Reading from GetObject reader failed", err) + return + } + + if len(newestContent) == 0 || !bytes.Equal(oldestContent, newestContent) { + logError(testName, function, args, startTime, "", "Unexpected destination object content", err) + return + } + + // Delete all objects and their versions as long as the bucket itself + if err = cleanupVersionedBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +func testComposeObjectWithVersioning() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "ComposeObject()" + args := map[string]interface{}{} + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) + if err != nil { + logError(testName, function, args, startTime, "", "Make bucket failed", err) + return + } + + err = c.EnableVersioning(context.Background(), bucketName) + if err != nil { + logError(testName, function, args, startTime, "", "Enable versioning failed", err) + return + } + + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + // var testFiles = []string{"datafile-5-MB", "datafile-10-kB"} + var testFiles = []string{"datafile-5-MB", "datafile-10-kB"} + var testFilesBytes [][]byte + + for _, testFile := range testFiles { + r := getDataReader(testFile) + buf, err := ioutil.ReadAll(r) + if err != nil { + logError(testName, function, args, startTime, "", "unexpected failure", err) + return + } + r.Close() + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + testFilesBytes = append(testFilesBytes, buf) + } + + objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) + + var results []minio.ObjectInfo + for info := range objectsInfo { + if info.Err != nil { + logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) + return + } + results = append(results, info) + } + + sort.SliceStable(results, func(i, j int) bool { + return results[i].Size > results[j].Size + }) + + // Source objects to concatenate. We also specify decryption + // key for each + src1 := minio.NewSourceInfo(bucketName, objectName, nil) + src1.SetVersionID(results[0].VersionID) + src2 := minio.NewSourceInfo(bucketName, objectName, nil) + src2.SetVersionID(results[1].VersionID) + + // Create slice of sources. + srcs := []minio.SourceInfo{src1, src2} + + // Create destination info + dst, err := minio.NewDestinationInfo(bucketName, objectName+"-copy", minio.DestInfoOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) + return + } + + _, err = c.ComposeObject(context.Background(), dst, srcs) + if err != nil { + logError(testName, function, args, startTime, "", "ComposeObject failed", err) + return + } + + // Destination object + readerCopy, err := c.GetObject(context.Background(), bucketName, objectName+"-copy", minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject of the copy object failed", err) + return + } + defer readerCopy.Close() + + copyContentBytes, err := ioutil.ReadAll(readerCopy) + if err != nil { + logError(testName, function, args, startTime, "", "Reading from the copy object reader failed", err) + return + } + + var expectedContent []byte + for _, fileBytes := range testFilesBytes { + expectedContent = append(expectedContent, fileBytes...) + } + + if len(copyContentBytes) == 0 || !bytes.Equal(copyContentBytes, expectedContent) { + logError(testName, function, args, startTime, "", "Unexpected destination object content", err) + return + } + + // Delete all objects and their versions as long as the bucket itself + if err = cleanupVersionedBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +func testRemoveObjectWithVersioning() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "DeleteObject()" + args := map[string]interface{}{} + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) + if err != nil { + logError(testName, function, args, startTime, "", "Make bucket failed", err) + return + } + + err = c.EnableVersioning(context.Background(), bucketName) + if err != nil { + logError(testName, function, args, startTime, "", "Enable versioning failed", err) + return + } + + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + _, err = c.PutObject(context.Background(), bucketName, objectName, getDataReader("datafile-10-kB"), int64(dataFileMap["datafile-10-kB"]), minio.PutObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) + var version minio.ObjectInfo + for info := range objectsInfo { + if info.Err != nil { + logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) + return + } + version = info + break + } + + err = c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{VersionID: version.VersionID}) + if err != nil { + logError(testName, function, args, startTime, "", "DeleteObject failed", err) + return + } + + objectsInfo = c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) + for range objectsInfo { + logError(testName, function, args, startTime, "", "Unexpected versioning info, should not have any one ", err) + return + } + + err = c.RemoveBucket(context.Background(), bucketName) + if err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +func testRemoveObjectsWithVersioning() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "DeleteObjects()" + args := map[string]interface{}{} + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) + if err != nil { + logError(testName, function, args, startTime, "", "Make bucket failed", err) + return + } + + err = c.EnableVersioning(context.Background(), bucketName) + if err != nil { + logError(testName, function, args, startTime, "", "Enable versioning failed", err) + return + } + + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + _, err = c.PutObject(context.Background(), bucketName, objectName, getDataReader("datafile-10-kB"), int64(dataFileMap["datafile-10-kB"]), minio.PutObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + objectsVersions := make(chan minio.ObjectVersion) + go func() { + objectsVersionsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) + for info := range objectsVersionsInfo { + if info.Err != nil { + logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) + return + } + objectsVersions <- minio.ObjectVersion{ + Key: info.Key, + VersionID: info.VersionID, + } + } + close(objectsVersions) + }() + + removeErrors := c.RemoveObjectsWithVersions(context.Background(), bucketName, objectsVersions, minio.RemoveObjectsOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "DeleteObjects call failed", err) + return + } + + for e := range removeErrors { + if e.Err != nil { + logError(testName, function, args, startTime, "", "Single delete operation failed", err) + return + } + } + + objectsVersionsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) + for range objectsVersionsInfo { + logError(testName, function, args, startTime, "", "Unexpected versioning info, should not have any one ", err) + return + } + + err = c.RemoveBucket(context.Background(), bucketName) + if err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +func testObjectTaggingWithVersioning() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "{Get,Set,Remove}ObjectTagging()" + args := map[string]interface{}{} + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } - bufSize := dataFileMap["datafile-129-MB"] - var reader = getDataReader("datafile-129-MB") - defer reader.Close() + err = c.EnableVersioning(context.Background(), bucketName) + if err != nil { + logError(testName, function, args, startTime, "", "Enable versioning failed", err) + return + } - // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName - // Object content type - objectContentType := "binary/octet-stream" - args["objectContentType"] = objectContentType + for _, file := range []string{"datafile-1-b", "datafile-10-kB"} { + _, err = c.PutObject(context.Background(), bucketName, objectName, getDataReader(file), int64(dataFileMap[file]), minio.PutObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + } + + versionsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) + + var versions []minio.ObjectInfo + for info := range versionsInfo { + if info.Err != nil { + logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) + return + } + versions = append(versions, info) + } + + sort.SliceStable(versions, func(i, j int) bool { + return versions[i].Size < versions[j].Size + }) - n, err := c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: objectContentType}) + tagsV1 := map[string]string{"key1": "val1"} + err = c.PutObjectTagging(context.Background(), bucketName, objectName, tagsV1, minio.PutObjectTaggingOptions{VersionID: versions[0].VersionID}) if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) + logError(testName, function, args, startTime, "", "PutObjectTaggingWithOptions (1) failed", err) return } - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match, expected "+string(bufSize)+" got "+string(n), err) + tagsV2 := map[string]string{"key2": "val2"} + err = c.PutObjectTagging(context.Background(), bucketName, objectName, tagsV2, minio.PutObjectTaggingOptions{VersionID: versions[1].VersionID}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObjectTaggingWithOptions (2) failed", err) return } - // Read the data back - r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) + tagsEqual := func(tags1, tags2 map[string]string) bool { + for k1, v1 := range tags1 { + v2, found := tags2[k1] + if found { + if v1 != v2 { + return false + } + } + } + return true + } + + gotTagsV1, err := c.GetObjectTagging(context.Background(), bucketName, objectName, minio.GetObjectTaggingOptions{VersionID: versions[0].VersionID}) if err != nil { - logError(testName, function, args, startTime, "", "Get Object failed", err) + logError(testName, function, args, startTime, "", "GetObjectTaggingWithOptions failed", err) return } - st, err := r.Stat() + if !tagsEqual(tagsV1, gotTagsV1) { + logError(testName, function, args, startTime, "", "Unexpected tags content (1)", err) + return + } + + gotTagsV2, err := c.GetObjectTagging(context.Background(), bucketName, objectName, minio.GetObjectTaggingOptions{}) if err != nil { - logError(testName, function, args, startTime, "", "Stat Object failed", err) + logError(testName, function, args, startTime, "", "GetObjectTaggingWithContext failed", err) return } - if st.Size != int64(bufSize) { - logError(testName, function, args, startTime, "", fmt.Sprintf("Number of bytes in stat does not match, expected %d got %d", bufSize, st.Size), err) + + if !tagsEqual(tagsV2, gotTagsV2) { + logError(testName, function, args, startTime, "", "Unexpected tags content (2)", err) return } - if st.ContentType != objectContentType && st.ContentType != "application/octet-stream" { - logError(testName, function, args, startTime, "", "Content types don't match", err) + + err = c.RemoveObjectTagging(context.Background(), bucketName, objectName, minio.RemoveObjectTaggingOptions{VersionID: versions[0].VersionID}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObjectTaggingWithOptions (2) failed", err) return } - if err := r.Close(); err != nil { - logError(testName, function, args, startTime, "", "Object Close failed", err) + + emptyTags, err := c.GetObjectTagging(context.Background(), bucketName, objectName, + minio.GetObjectTaggingOptions{VersionID: versions[0].VersionID}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObjectTaggingWithOptions failed", err) return } - if err := r.Close(); err == nil { - logError(testName, function, args, startTime, "", "Object is already closed, didn't return error on Close", err) + + if len(emptyTags) != 0 { + logError(testName, function, args, startTime, "", "Unexpected tags content (2)", err) return } - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { + // Delete all objects and their versions as long as the bucket itself + if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "Cleanup failed", err) return } @@ -657,18 +1672,13 @@ func testPutObjectWithMetadata() { "X-Amz-Meta-CustomKey": {"extra spaces in value"}, } - n, err := c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ + _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ ContentType: customContentType}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match, expected "+string(bufSize)+" got "+string(n), err) - return - } - // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { @@ -752,7 +1762,7 @@ func testPutObjectWithContentLanguage() { } data := bytes.Repeat([]byte("a"), int(0)) - n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(data), int64(0), minio.PutObjectOptions{ + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(data), int64(0), minio.PutObjectOptions{ ContentLanguage: "en", }) if err != nil { @@ -760,11 +1770,6 @@ func testPutObjectWithContentLanguage() { return } - if n != 0 { - logError(testName, function, args, startTime, "", "Expected upload object '0' doesn't match with PutObject return value", err) - return - } - objInfo, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) @@ -835,16 +1840,27 @@ func testPutObjectStreaming() { for _, size := range sizes { data := bytes.Repeat([]byte("a"), int(size)) - n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(data), int64(size), minio.PutObjectOptions{}) + ui, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(data), int64(size), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectStreaming failed", err) return } - if n != size { - logError(testName, function, args, startTime, "", "Expected upload object size doesn't match with PutObjectStreaming return value", err) + if ui.Size != size { + logError(testName, function, args, startTime, "", "PutObjectStreaming result has unexpected size", nil) return } + + objInfo, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject failed", err) + return + } + if objInfo.Size != size { + logError(testName, function, args, startTime, "", "Unexpected size", err) + return + } + } // Delete all objects and buckets @@ -911,17 +1927,12 @@ func testGetObjectSeekEnd() { return } - n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes read does not match, expected "+string(int64(bufSize))+" got "+string(n), err) - return - } - // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { @@ -1037,17 +2048,12 @@ func testGetObjectClosedTwice() { objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName - n, err := c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "PutObject response doesn't match sent bytes, expected "+string(int64(bufSize))+" got "+string(n), err) - return - } - // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { @@ -1146,7 +2152,7 @@ func testRemoveObjectsWithContext() { defer cancel() // Call RemoveObjects API with short timeout. - errorCh := c.RemoveObjects(ctx, bucketName, objectsCh) + errorCh := c.RemoveObjects(ctx, bucketName, objectsCh, minio.RemoveObjectsOptions{}) // Check for error. select { case r := <-errorCh: @@ -1160,7 +2166,7 @@ func testRemoveObjectsWithContext() { args["ctx"] = ctx defer cancel() // Perform RemoveObjects with the longer timeout. Expect the removals to succeed. - errorCh = c.RemoveObjects(ctx, bucketName, objectsCh) + errorCh = c.RemoveObjects(ctx, bucketName, objectsCh, minio.RemoveObjectsOptions{}) select { case r, more := <-errorCh: if more || r.Err != nil { @@ -1242,7 +2248,7 @@ func testRemoveMultipleObjects() { }() // Call RemoveObjects API - errorCh := c.RemoveObjects(context.Background(), bucketName, objectsCh) + errorCh := c.RemoveObjects(context.Background(), bucketName, objectsCh, minio.RemoveObjectsOptions{}) // Check if errorCh doesn't receive any error select { @@ -1337,15 +2343,11 @@ func testFPutObjectMultipart() { args["objectContentType"] = objectContentType // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) - n, err := c.FPutObject(context.Background(), bucketName, objectName, fileName, minio.PutObjectOptions{ContentType: objectContentType}) + _, err = c.FPutObject(context.Background(), bucketName, objectName, fileName, minio.PutObjectOptions{ContentType: objectContentType}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject failed", err) return } - if n != int64(totalSize) { - logError(testName, function, args, startTime, "", "FPutObject failed", err) - return - } r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { @@ -1448,7 +2450,6 @@ func testFPutObject() { defer os.Remove(file.Name()) fName = file.Name() } - totalSize := dataFileMap["datafile-129-MB"] // Set base object name function = "FPutObject(bucketName, objectName, fileName, opts)" @@ -1458,28 +2459,25 @@ func testFPutObject() { args["opts"] = minio.PutObjectOptions{ContentType: "application/octet-stream"} // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) - n, err := c.FPutObject(context.Background(), bucketName, objectName+"-standard", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) - + ui, err := c.FPutObject(context.Background(), bucketName, objectName+"-standard", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject failed", err) return } - if n != int64(totalSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(totalSize)+", got "+string(n), err) + + if ui.Size != int64(dataFileMap["datafile-129-MB"]) { + logError(testName, function, args, startTime, "", "FPutObject returned an unexpected upload size", err) return } // Perform FPutObject with no contentType provided (Expecting application/octet-stream) args["objectName"] = objectName + "-Octet" - n, err = c.FPutObject(context.Background(), bucketName, objectName+"-Octet", fName, minio.PutObjectOptions{}) + _, err = c.FPutObject(context.Background(), bucketName, objectName+"-Octet", fName, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "File close failed", err) return } - if n != int64(totalSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(totalSize)+", got "+string(n), err) - return - } + srcFile, err := os.Open(fName) if err != nil { logError(testName, function, args, startTime, "", "File open failed", err) @@ -1502,15 +2500,11 @@ func testFPutObject() { // Perform FPutObject with no contentType provided (Expecting application/x-gtar) args["objectName"] = objectName + "-GTar" args["opts"] = minio.PutObjectOptions{} - n, err = c.FPutObject(context.Background(), bucketName, objectName+"-GTar", fName+".gtar", minio.PutObjectOptions{}) + _, err = c.FPutObject(context.Background(), bucketName, objectName+"-GTar", fName+".gtar", minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject failed", err) return } - if n != int64(totalSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(totalSize)+", got "+string(n), err) - return - } // Check headers function = "StatObject(bucketName, objectName, opts)" @@ -1627,7 +2621,6 @@ func testFPutObjectWithContext() { defer os.Remove(file.Name()) fName = file.Name() } - totalSize := dataFileMap["datafile-1-MB"] // Set base object name objectName := bucketName + "FPutObjectWithContext" @@ -1645,15 +2638,11 @@ func testFPutObjectWithContext() { ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) defer cancel() // Perform FPutObject with a long timeout. Expect the put object to succeed - n, err := c.FPutObject(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{}) + _, err = c.FPutObject(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject shouldn't fail on long timeout", err) return } - if n != int64(totalSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(totalSize)+", got "+string(n), err) - return - } _, err = c.StatObject(context.Background(), bucketName, objectName+"-Longtimeout", minio.StatObjectOptions{}) if err != nil { @@ -1739,7 +2728,6 @@ func testFPutObjectWithContextV2() { defer os.Remove(file.Name()) fName = file.Name() } - totalSize := dataFileMap["datafile-1-MB"] // Set base object name objectName := bucketName + "FPutObjectWithContext" @@ -1758,15 +2746,11 @@ func testFPutObjectWithContextV2() { ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) defer cancel() // Perform FPutObject with a long timeout. Expect the put object to succeed - n, err := c.FPutObject(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{}) + _, err = c.FPutObject(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject shouldn't fail on longer timeout", err) return } - if n != int64(totalSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match:wanted"+string(totalSize)+" got "+string(n), err) - return - } _, err = c.StatObject(context.Background(), bucketName, objectName+"-Longtimeout", minio.StatObjectOptions{}) if err != nil { @@ -1925,17 +2909,12 @@ func testGetObjectReadSeekFunctional() { } // Save the data - n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) - return - } - // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { @@ -2092,17 +3071,12 @@ func testGetObjectReadAtFunctional() { } // Save the data - n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) - return - } - // read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { @@ -2187,7 +3161,7 @@ func testGetObjectReadAtFunctional() { return } - buf5 := make([]byte, n) + buf5 := make([]byte, len(buf)) // Read the whole object. m, err = r.ReadAt(buf5, 0) if err != nil { @@ -2205,7 +3179,7 @@ func testGetObjectReadAtFunctional() { return } - buf6 := make([]byte, n+1) + buf6 := make([]byte, len(buf)+1) // Read the whole object and beyond. _, err = r.ReadAt(buf6, 0) if err != nil { @@ -2277,17 +3251,12 @@ func testGetObjectReadAtWhenEOFWasReached() { } // Save the data - n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) - return - } - // read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { @@ -2296,7 +3265,7 @@ func testGetObjectReadAtWhenEOFWasReached() { } // read directly - buf1 := make([]byte, n) + buf1 := make([]byte, len(buf)) buf2 := make([]byte, 512) m, err := r.Read(buf1) @@ -2391,7 +3360,6 @@ func testPresignedPostPolicy() { } // Generate 33K of data. - bufSize := dataFileMap["datafile-33-kB"] var reader = getDataReader("datafile-33-kB") defer reader.Close() @@ -2407,17 +3375,12 @@ func testPresignedPostPolicy() { } // Save the data - n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+" got "+string(n), err) - return - } - policy := minio.NewPostPolicy() if err := policy.SetBucket(""); err == nil { @@ -2625,17 +3588,12 @@ func testCopyObject() { // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) - return - } - r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) @@ -2687,7 +3645,7 @@ func testCopyObject() { return } - dst, err := minio.NewDestinationInfo(bucketName+"-copy", objectName+"-copy", nil, nil) + dst, err := minio.NewDestinationInfo(bucketName+"-copy", objectName+"-copy", minio.DestInfoOptions{}) args["dst"] = dst if err != nil { logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) @@ -2695,12 +3653,17 @@ func testCopyObject() { } // Perform the Copy - err = c.CopyObject(context.Background(), dst, src) + ui, err := c.CopyObject(context.Background(), dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } + if ui.Size != 0 { + logError(testName, function, args, startTime, "", "CopyObject returned unexpeced size", nil) + return + } + // Source object r, err = c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { @@ -2748,7 +3711,7 @@ func testCopyObject() { } // Perform the Copy which should fail - err = c.CopyObject(context.Background(), dst, src) + _, err = c.CopyObject(context.Background(), dst, src) if err == nil { logError(testName, function, args, startTime, "", "CopyObject did not fail for invalid conditions", err) return @@ -2756,9 +3719,13 @@ func testCopyObject() { // Perform the Copy which should update only metadata. src = minio.NewSourceInfo(bucketName, objectName, nil) - dst, err = minio.NewDestinationInfo(bucketName, objectName, nil, map[string]string{ - "Copy": "should be same", - }) + dst, err = minio.NewDestinationInfo(bucketName, objectName, + minio.DestInfoOptions{ + UserMeta: map[string]string{ + "Copy": "should be same", + }, + }, + ) args["dst"] = dst args["src"] = src if err != nil { @@ -2766,7 +3733,7 @@ func testCopyObject() { return } - err = c.CopyObject(context.Background(), dst, src) + _, err = c.CopyObject(context.Background(), dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject shouldn't fail", err) return @@ -2866,7 +3833,7 @@ func testSSECEncryptedGetObjectReadSeekFunctional() { } // Save the data - n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ ContentType: "binary/octet-stream", ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), }) @@ -2875,11 +3842,6 @@ func testSSECEncryptedGetObjectReadSeekFunctional() { return } - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) - return - } - // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{ ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), @@ -3054,7 +4016,7 @@ func testSSES3EncryptedGetObjectReadSeekFunctional() { } // Save the data - n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ ContentType: "binary/octet-stream", ServerSideEncryption: encrypt.NewSSE(), }) @@ -3063,11 +4025,6 @@ func testSSES3EncryptedGetObjectReadSeekFunctional() { return } - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) - return - } - // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { @@ -3232,7 +4189,7 @@ func testSSECEncryptedGetObjectReadAtFunctional() { } // Save the data - n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ ContentType: "binary/octet-stream", ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), }) @@ -3241,11 +4198,6 @@ func testSSECEncryptedGetObjectReadAtFunctional() { return } - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) - return - } - // read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{ ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), @@ -3333,7 +4285,7 @@ func testSSECEncryptedGetObjectReadAtFunctional() { return } - buf5 := make([]byte, n) + buf5 := make([]byte, len(buf)) // Read the whole object. m, err = r.ReadAt(buf5, 0) if err != nil { @@ -3351,7 +4303,7 @@ func testSSECEncryptedGetObjectReadAtFunctional() { return } - buf6 := make([]byte, n+1) + buf6 := make([]byte, len(buf)+1) // Read the whole object and beyond. _, err = r.ReadAt(buf6, 0) if err != nil { @@ -3423,7 +4375,7 @@ func testSSES3EncryptedGetObjectReadAtFunctional() { } // Save the data - n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ ContentType: "binary/octet-stream", ServerSideEncryption: encrypt.NewSSE(), }) @@ -3432,11 +4384,6 @@ func testSSES3EncryptedGetObjectReadAtFunctional() { return } - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) - return - } - // read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { @@ -3522,7 +4469,7 @@ func testSSES3EncryptedGetObjectReadAtFunctional() { return } - buf5 := make([]byte, n) + buf5 := make([]byte, len(buf)) // Read the whole object. m, err = r.ReadAt(buf5, 0) if err != nil { @@ -3540,7 +4487,7 @@ func testSSES3EncryptedGetObjectReadAtFunctional() { return } - buf6 := make([]byte, n+1) + buf6 := make([]byte, len(buf)+1) // Read the whole object and beyond. _, err = r.ReadAt(buf6, 0) if err != nil { @@ -4360,34 +5307,24 @@ func testFunctional() { "contentType": "", } - n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } - if n != int64(len(buf)) { - logError(testName, function, args, startTime, "", "Length doesn't match, expected "+string(int64(len(buf)))+" got "+string(n), err) - return - } - args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName + "-nolength", "contentType": "binary/octet-stream", } - n, err = c.PutObject(context.Background(), bucketName, objectName+"-nolength", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + _, err = c.PutObject(context.Background(), bucketName, objectName+"-nolength", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } - if n != int64(len(buf)) { - logError(testName, function, args, startTime, "", "Length doesn't match, expected "+string(int64(len(buf)))+" got "+string(n), err) - return - } - // Instantiate a done channel to close all listing. doneCh := make(chan struct{}) defer close(doneCh) @@ -4403,7 +5340,7 @@ func testFunctional() { "isRecursive": isRecursive, } - for obj := range c.ListObjects(context.Background(), bucketName, objectName, isRecursive, doneCh) { + for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{ForceV1: true, Prefix: objectName, Recursive: true}) { if obj.Key == objectName { objFound = true break @@ -4416,7 +5353,7 @@ func testFunctional() { objFound = false isRecursive = true // Recursive is true. - function = "ListObjectsV2(bucketName, objectName, isRecursive, doneCh)" + function = "ListObjects()" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, @@ -4424,7 +5361,7 @@ func testFunctional() { "isRecursive": isRecursive, } - for obj := range c.ListObjectsV2(context.Background(), bucketName, objectName, isRecursive, doneCh) { + for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{Prefix: objectName, Recursive: isRecursive}) { if obj.Key == objectName { objFound = true break @@ -4723,14 +5660,14 @@ func testFunctional() { "bucketName": bucketName, "objectName": objectName, } - err = c.RemoveObject(context.Background(), bucketName, objectName) + err = c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "RemoveObject failed", err) return } args["objectName"] = objectName + "-f" - err = c.RemoveObject(context.Background(), bucketName, objectName+"-f") + err = c.RemoveObject(context.Background(), bucketName, objectName+"-f", minio.RemoveObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "RemoveObject failed", err) @@ -4738,7 +5675,7 @@ func testFunctional() { } args["objectName"] = objectName + "-nolength" - err = c.RemoveObject(context.Background(), bucketName, objectName+"-nolength") + err = c.RemoveObject(context.Background(), bucketName, objectName+"-nolength", minio.RemoveObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "RemoveObject failed", err) @@ -4746,7 +5683,7 @@ func testFunctional() { } args["objectName"] = objectName + "-presigned" - err = c.RemoveObject(context.Background(), bucketName, objectName+"-presigned") + err = c.RemoveObject(context.Background(), bucketName, objectName+"-presigned", minio.RemoveObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "RemoveObject failed", err) @@ -4828,7 +5765,7 @@ func testGetObjectModified() { return } - defer c.RemoveObject(context.Background(), bucketName, objectName) + defer c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{}) reader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { @@ -4956,15 +5893,11 @@ func testPutObjectUploadSeekedObject() { return } - n, err := c.PutObject(context.Background(), bucketName, objectName, tempfile, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + _, err = c.PutObject(context.Background(), bucketName, objectName, tempfile, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } - if n != int64(length-offset) { - logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid length returned, expected %d got %d", int64(length-offset), n), err) - return - } tempfile.Close() obj, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) @@ -4974,7 +5907,7 @@ func testPutObjectUploadSeekedObject() { } defer obj.Close() - n, err = obj.Seek(int64(offset), 0) + n, err := obj.Seek(int64(offset), 0) if err != nil { logError(testName, function, args, startTime, "", "Seek failed", err) return @@ -4984,12 +5917,17 @@ func testPutObjectUploadSeekedObject() { return } - n, err = c.PutObject(context.Background(), bucketName, objectName+"getobject", obj, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + _, err = c.PutObject(context.Background(), bucketName, objectName+"getobject", obj, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } - if n != int64(length-offset) { + st, err := c.StatObject(context.Background(), bucketName, objectName+"getobject", minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject failed", err) + return + } + if st.Size != int64(length-offset) { logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid offset returned, expected %d got %d", int64(length-offset), n), err) return } @@ -5121,17 +6059,12 @@ func testGetObjectClosedTwiceV2() { objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName - n, err := c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(n), err) - return - } - // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { @@ -5243,29 +6176,21 @@ func testFPutObjectV2() { args["fileName"] = file.Name() // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) - n, err = c.FPutObject(context.Background(), bucketName, objectName+"-standard", file.Name(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) + _, err = c.FPutObject(context.Background(), bucketName, objectName+"-standard", file.Name(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject failed", err) return } - if n != int64(11*1024*1024) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) - return - } // Perform FPutObject with no contentType provided (Expecting application/octet-stream) args["objectName"] = objectName + "-Octet" args["contentType"] = "" - n, err = c.FPutObject(context.Background(), bucketName, objectName+"-Octet", file.Name(), minio.PutObjectOptions{}) + _, err = c.FPutObject(context.Background(), bucketName, objectName+"-Octet", file.Name(), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject failed", err) return } - if n != int64(11*1024*1024) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) - return - } // Add extension to temp file name fileName := file.Name() @@ -5280,22 +6205,24 @@ func testFPutObjectV2() { args["contentType"] = "" args["fileName"] = fileName + ".gtar" - n, err = c.FPutObject(context.Background(), bucketName, objectName+"-GTar", fileName+".gtar", minio.PutObjectOptions{}) + _, err = c.FPutObject(context.Background(), bucketName, objectName+"-GTar", fileName+".gtar", minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject failed", err) return } - if n != int64(11*1024*1024) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) - return - } - // Check headers + // Check headers and sizes rStandard, err := c.StatObject(context.Background(), bucketName, objectName+"-standard", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } + + if rStandard.Size != 11*1024*1024 { + logError(testName, function, args, startTime, "", "Unexpected size", nil) + return + } + if rStandard.ContentType != "application/octet-stream" { logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/octet-stream , got "+rStandard.ContentType, err) return @@ -5311,11 +6238,20 @@ func testFPutObjectV2() { return } + if rOctet.Size != 11*1024*1024 { + logError(testName, function, args, startTime, "", "Unexpected size", nil) + return + } + rGTar, err := c.StatObject(context.Background(), bucketName, objectName+"-GTar", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } + if rGTar.Size != 11*1024*1024 { + logError(testName, function, args, startTime, "", "Unexpected size", nil) + return + } if rGTar.ContentType != "application/x-gtar" && rGTar.ContentType != "application/octet-stream" { logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/x-gtar , got "+rGTar.ContentType, err) return @@ -5457,17 +6393,12 @@ func testGetObjectReadSeekFunctionalV2() { } // Save the data. - n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+" got "+string(n), err) - return - } - // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { @@ -5488,7 +6419,7 @@ func testGetObjectReadSeekFunctionalV2() { } offset := int64(2048) - n, err = r.Seek(offset, 0) + n, err := r.Seek(offset, 0) if err != nil { logError(testName, function, args, startTime, "", "Seek failed", err) return @@ -5621,17 +6552,12 @@ func testGetObjectReadAtFunctionalV2() { } // Save the data - n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(n), err) - return - } - // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { @@ -5700,7 +6626,7 @@ func testGetObjectReadAtFunctionalV2() { return } - buf5 := make([]byte, n) + buf5 := make([]byte, bufSize) // Read the whole object. m, err = r.ReadAt(buf5, 0) if err != nil { @@ -5718,7 +6644,7 @@ func testGetObjectReadAtFunctionalV2() { return } - buf6 := make([]byte, n+1) + buf6 := make([]byte, bufSize+1) // Read the whole object and beyond. _, err = r.ReadAt(buf6, 0) if err != nil { @@ -5789,17 +6715,12 @@ func testCopyObjectV2() { // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+" got "+string(n), err) - return - } - r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) @@ -5852,7 +6773,7 @@ func testCopyObjectV2() { return } - dst, err := minio.NewDestinationInfo(bucketName+"-copy", objectName+"-copy", nil, nil) + dst, err := minio.NewDestinationInfo(bucketName+"-copy", objectName+"-copy", minio.DestInfoOptions{}) args["destination"] = dst if err != nil { logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) @@ -5860,7 +6781,7 @@ func testCopyObjectV2() { } // Perform the Copy - err = c.CopyObject(context.Background(), dst, src) + _, err = c.CopyObject(context.Background(), dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return @@ -5912,7 +6833,7 @@ func testCopyObjectV2() { } // Perform the Copy which should fail - err = c.CopyObject(context.Background(), dst, src) + _, err = c.CopyObject(context.Background(), dst, src) if err == nil { logError(testName, function, args, startTime, "", "CopyObject did not fail for invalid conditions", err) return @@ -5952,7 +6873,7 @@ func testComposeObjectErrorCasesWrapper(c *minio.Client) { // concatenated. srcArr := [10001]minio.SourceInfo{} srcSlice := srcArr[:] - dst, err := minio.NewDestinationInfo(bucketName, "object", nil, nil) + dst, err := minio.NewDestinationInfo(bucketName, "object", minio.DestInfoOptions{}) if err != nil { logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) return @@ -5962,7 +6883,7 @@ func testComposeObjectErrorCasesWrapper(c *minio.Client) { // Just explain about srcArr in args["sourceList"] // to stop having 10,001 null headers logged args["sourceList"] = "source array of 10,001 elements" - if err := c.ComposeObject(context.Background(), dst, srcSlice); err == nil { + if _, err := c.ComposeObject(context.Background(), dst, srcSlice); err == nil { logError(testName, function, args, startTime, "", "Expected error in ComposeObject", err) return } else if err.Error() != "There must be as least one and up to 10000 source objects." { @@ -5989,7 +6910,7 @@ func testComposeObjectErrorCasesWrapper(c *minio.Client) { return } // 3. ComposeObject call should fail. - if err := c.ComposeObject(context.Background(), dst, []minio.SourceInfo{badSrc}); err == nil { + if _, err := c.ComposeObject(context.Background(), dst, []minio.SourceInfo{badSrc}); err == nil { logError(testName, function, args, startTime, "", "ComposeObject expected to fail", err) return } else if !strings.Contains(err.Error(), "has invalid segment-to-copy") { @@ -6070,19 +6991,24 @@ func testComposeMultipleSources(c *minio.Client) { } args["sourceList"] = srcs - dst, err := minio.NewDestinationInfo(bucketName, "dstObject", nil, nil) + dst, err := minio.NewDestinationInfo(bucketName, "dstObject", minio.DestInfoOptions{}) args["destination"] = dst if err != nil { logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) return } - err = c.ComposeObject(context.Background(), dst, srcs) + ui, err := c.ComposeObject(context.Background(), dst, srcs) if err != nil { logError(testName, function, args, startTime, "", "ComposeObject failed", err) return } + if ui.Size != 9*srcSize+1 { + logError(testName, function, args, startTime, "", "ComposeObject returned unexpected size", err) + return + } + objProps, err := c.StatObject(context.Background(), bucketName, "dstObject", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) @@ -6166,7 +7092,7 @@ func testEncryptedEmptyObject() { } // 2. Test CopyObject for an empty object - dstInfo, err := minio.NewDestinationInfo(bucketName, "new-object", sse, nil) + dstInfo, err := minio.NewDestinationInfo(bucketName, "new-object", minio.DestInfoOptions{Encryption: sse}) if err != nil { args["objectName"] = "new-object" function = "NewDestinationInfo(bucketName, objectName, sse, userMetadata)" @@ -6174,7 +7100,7 @@ func testEncryptedEmptyObject() { return } srcInfo := minio.NewSourceInfo(bucketName, "object", sse) - if err = c.CopyObject(context.Background(), dstInfo, srcInfo); err != nil { + if _, err = c.CopyObject(context.Background(), dstInfo, srcInfo); err != nil { function = "CopyObject(dstInfo, srcInfo)" logError(testName, function, map[string]interface{}{}, startTime, "", "CopyObject failed", err) return @@ -6182,7 +7108,7 @@ func testEncryptedEmptyObject() { // 3. Test Key rotation newSSE := encrypt.DefaultPBKDF([]byte("Don't Panic"), []byte(bucketName+"new-object")) - dstInfo, err = minio.NewDestinationInfo(bucketName, "new-object", newSSE, nil) + dstInfo, err = minio.NewDestinationInfo(bucketName, "new-object", minio.DestInfoOptions{Encryption: newSSE}) if err != nil { args["objectName"] = "new-object" function = "NewDestinationInfo(bucketName, objectName, encryptSSEC, userMetadata)" @@ -6191,7 +7117,7 @@ func testEncryptedEmptyObject() { } srcInfo = minio.NewSourceInfo(bucketName, "new-object", sse) - if err = c.CopyObject(context.Background(), dstInfo, srcInfo); err != nil { + if _, err = c.CopyObject(context.Background(), dstInfo, srcInfo); err != nil { function = "CopyObject(dstInfo, srcInfo)" logError(testName, function, map[string]interface{}{}, startTime, "", "CopyObject with key rotation failed", err) return @@ -6257,14 +7183,14 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, // 2. copy object and change encryption key src := minio.NewSourceInfo(bucketName, "srcObject", srcEncryption) args["source"] = src - dst, err := minio.NewDestinationInfo(bucketName, "dstObject", sseDst, nil) + dst, err := minio.NewDestinationInfo(bucketName, "dstObject", minio.DestInfoOptions{Encryption: sseDst}) if err != nil { logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) return } args["destination"] = dst - err = c.CopyObject(context.Background(), dst, src) + _, err = c.CopyObject(context.Background(), dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return @@ -6301,14 +7227,14 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, newSSE = encrypt.NewSSE() } if newSSE != nil { - dst, err = minio.NewDestinationInfo(bucketName, "srcObject", newSSE, nil) + dst, err = minio.NewDestinationInfo(bucketName, "srcObject", minio.DestInfoOptions{Encryption: newSSE}) if err != nil { logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) return } args["destination"] = dst - err = c.CopyObject(context.Background(), dst, src) + _, err = c.CopyObject(context.Background(), dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return @@ -6332,7 +7258,7 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, } reader.Close() // Test in-place decryption. - dst, err = minio.NewDestinationInfo(bucketName, "srcObject", nil, nil) + dst, err = minio.NewDestinationInfo(bucketName, "srcObject", minio.DestInfoOptions{}) if err != nil { logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) return @@ -6341,7 +7267,7 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, src = minio.NewSourceInfo(bucketName, "srcObject", newSSE) args["source"] = src - err = c.CopyObject(context.Background(), dst, src) + _, err = c.CopyObject(context.Background(), dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject Key rotation failed", err) return @@ -6690,14 +7616,14 @@ func testDecryptedCopyObject() { src := minio.NewSourceInfo(bucketName, objectName, encrypt.SSECopy(encryption)) args["source"] = src - dst, err := minio.NewDestinationInfo(bucketName, "decrypted-"+objectName, nil, nil) + dst, err := minio.NewDestinationInfo(bucketName, "decrypted-"+objectName, minio.DestInfoOptions{}) if err != nil { logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) return } args["destination"] = dst - if err = c.CopyObject(context.Background(), dst, src); err != nil { + if _, err = c.CopyObject(context.Background(), dst, src); err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } @@ -6931,13 +7857,18 @@ func testSSECEncryptedToSSECCopyObjectPart() { UserMetadata: putmetadata, ServerSideEncryption: srcencryption, } - objInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) + uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } - if objInfo.Size != int64(len(buf)) { - logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) + st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: srcencryption}}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject call failed", err) + } + + if st.Size != int64(len(buf)) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) } destBucketName := bucketName @@ -6960,7 +7891,7 @@ func testSSECEncryptedToSSECCopyObjectPart() { metadata[k] = v[0] } - metadata["x-amz-copy-source-if-match"] = objInfo.ETag + metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) @@ -6987,7 +7918,7 @@ func testSSECEncryptedToSSECCopyObjectPart() { } // Stat the object and check its length matches - objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) + objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -7086,13 +8017,18 @@ func testSSECEncryptedToUnencryptedCopyPart() { }, ServerSideEncryption: srcencryption, } - objInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) + uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } - if objInfo.Size != int64(len(buf)) { - logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) + st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: srcencryption}}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject call failed", err) + } + + if st.Size != int64(len(buf)) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) } destBucketName := bucketName @@ -7114,7 +8050,7 @@ func testSSECEncryptedToUnencryptedCopyPart() { metadata[k] = v[0] } - metadata["x-amz-copy-source-if-match"] = objInfo.ETag + metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) @@ -7141,7 +8077,7 @@ func testSSECEncryptedToUnencryptedCopyPart() { } // Stat the object and check its length matches - objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) + objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -7241,13 +8177,18 @@ func testSSECEncryptedToSSES3CopyObjectPart() { ServerSideEncryption: srcencryption, } - objInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) + uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } - if objInfo.Size != int64(len(buf)) { - logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) + st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: srcencryption}}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject call failed", err) + } + + if st.Size != int64(len(buf)) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) } destBucketName := bucketName @@ -7271,7 +8212,7 @@ func testSSECEncryptedToSSES3CopyObjectPart() { metadata[k] = v[0] } - metadata["x-amz-copy-source-if-match"] = objInfo.ETag + metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) @@ -7298,7 +8239,7 @@ func testSSECEncryptedToSSES3CopyObjectPart() { } // Stat the object and check its length matches - objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) + objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -7395,13 +8336,18 @@ func testUnencryptedToSSECCopyObjectPart() { opts := minio.PutObjectOptions{ UserMetadata: putmetadata, } - objInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) + uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } - if objInfo.Size != int64(len(buf)) { - logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) + st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject call failed", err) + } + + if st.Size != int64(len(buf)) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) } destBucketName := bucketName @@ -7423,7 +8369,7 @@ func testUnencryptedToSSECCopyObjectPart() { metadata[k] = v[0] } - metadata["x-amz-copy-source-if-match"] = objInfo.ETag + metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) @@ -7450,7 +8396,7 @@ func testUnencryptedToSSECCopyObjectPart() { } // Stat the object and check its length matches - objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) + objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -7546,13 +8492,17 @@ func testUnencryptedToUnencryptedCopyPart() { opts := minio.PutObjectOptions{ UserMetadata: putmetadata, } - objInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) + uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } + st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject call failed", err) + } - if objInfo.Size != int64(len(buf)) { - logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) + if st.Size != int64(len(buf)) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) } destBucketName := bucketName @@ -7572,7 +8522,7 @@ func testUnencryptedToUnencryptedCopyPart() { metadata[k] = v[0] } - metadata["x-amz-copy-source-if-match"] = objInfo.ETag + metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) @@ -7599,7 +8549,7 @@ func testUnencryptedToUnencryptedCopyPart() { } // Stat the object and check its length matches - objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) + objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -7694,13 +8644,17 @@ func testUnencryptedToSSES3CopyObjectPart() { "Content-Type": "binary/octet-stream", }, } - objInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) + uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } + st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject call failed", err) + } - if objInfo.Size != int64(len(buf)) { - logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) + if st.Size != int64(len(buf)) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) } destBucketName := bucketName @@ -7723,7 +8677,7 @@ func testUnencryptedToSSES3CopyObjectPart() { metadata[k] = v[0] } - metadata["x-amz-copy-source-if-match"] = objInfo.ETag + metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) @@ -7750,7 +8704,7 @@ func testUnencryptedToSSES3CopyObjectPart() { } // Stat the object and check its length matches - objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) + objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -7848,13 +8802,18 @@ func testSSES3EncryptedToSSECCopyObjectPart() { }, ServerSideEncryption: srcEncryption, } - objInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) + uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } - if objInfo.Size != int64(len(buf)) { - logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) + st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: srcEncryption}}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject call failed", err) + } + + if st.Size != int64(len(buf)) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) } destBucketName := bucketName @@ -7876,7 +8835,7 @@ func testSSES3EncryptedToSSECCopyObjectPart() { metadata[k] = v[0] } - metadata["x-amz-copy-source-if-match"] = objInfo.ETag + metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) @@ -7903,7 +8862,7 @@ func testSSES3EncryptedToSSECCopyObjectPart() { } // Stat the object and check its length matches - objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) + objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -8000,13 +8959,17 @@ func testSSES3EncryptedToUnencryptedCopyPart() { }, ServerSideEncryption: srcEncryption, } - objInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) + uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } + st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: srcEncryption}}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject call failed", err) + } - if objInfo.Size != int64(len(buf)) { - logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) + if st.Size != int64(len(buf)) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) } destBucketName := bucketName @@ -8026,7 +8989,7 @@ func testSSES3EncryptedToUnencryptedCopyPart() { metadata[k] = v[0] } - metadata["x-amz-copy-source-if-match"] = objInfo.ETag + metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) @@ -8053,7 +9016,7 @@ func testSSES3EncryptedToUnencryptedCopyPart() { } // Stat the object and check its length matches - objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) + objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -8151,13 +9114,16 @@ func testSSES3EncryptedToSSES3CopyObjectPart() { ServerSideEncryption: srcEncryption, } - objInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) + uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } - - if objInfo.Size != int64(len(buf)) { - logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) + st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: srcEncryption}}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject call failed", err) + } + if st.Size != int64(len(buf)) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) } destBucketName := bucketName @@ -8180,7 +9146,7 @@ func testSSES3EncryptedToSSES3CopyObjectPart() { metadata[k] = v[0] } - metadata["x-amz-copy-source-if-match"] = objInfo.ETag + metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) @@ -8207,7 +9173,7 @@ func testSSES3EncryptedToSSES3CopyObjectPart() { } // Stat the object and check its length matches - objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) + objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -8330,7 +9296,7 @@ func testUserMetadataCopyingWrapper(c *minio.Client) { // 2. create source src := minio.NewSourceInfo(bucketName, "srcObject", nil) // 2.1 create destination with metadata set - dst1, err := minio.NewDestinationInfo(bucketName, "dstObject-1", nil, map[string]string{"notmyheader": "notmyvalue"}) + dst1, err := minio.NewDestinationInfo(bucketName, "dstObject-1", minio.DestInfoOptions{UserMeta: map[string]string{"notmyheader": "notmyvalue"}}) if err != nil { logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) return @@ -8340,7 +9306,7 @@ func testUserMetadataCopyingWrapper(c *minio.Client) { // the headers on the copy. args["source"] = src args["destination"] = dst1 - err = c.CopyObject(context.Background(), dst1, src) + _, err = c.CopyObject(context.Background(), dst1, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return @@ -8354,7 +9320,7 @@ func testUserMetadataCopyingWrapper(c *minio.Client) { } // 4. create destination with no metadata set and same source - dst2, err := minio.NewDestinationInfo(bucketName, "dstObject-2", nil, nil) + dst2, err := minio.NewDestinationInfo(bucketName, "dstObject-2", minio.DestInfoOptions{}) if err != nil { logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) return @@ -8365,7 +9331,7 @@ func testUserMetadataCopyingWrapper(c *minio.Client) { // copies metadata. args["source"] = src args["destination"] = dst2 - err = c.CopyObject(context.Background(), dst2, src) + _, err = c.CopyObject(context.Background(), dst2, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return @@ -8382,7 +9348,7 @@ func testUserMetadataCopyingWrapper(c *minio.Client) { minio.NewSourceInfo(bucketName, "srcObject", nil), minio.NewSourceInfo(bucketName, "srcObject", nil), } - dst3, err := minio.NewDestinationInfo(bucketName, "dstObject-3", nil, nil) + dst3, err := minio.NewDestinationInfo(bucketName, "dstObject-3", minio.DestInfoOptions{}) if err != nil { logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) return @@ -8391,7 +9357,7 @@ func testUserMetadataCopyingWrapper(c *minio.Client) { function = "ComposeObject(destination, sources)" args["source"] = srcs args["destination"] = dst3 - err = c.ComposeObject(context.Background(), dst3, srcs) + _, err = c.ComposeObject(context.Background(), dst3, srcs) if err != nil { logError(testName, function, args, startTime, "", "ComposeObject failed", err) return @@ -8408,7 +9374,7 @@ func testUserMetadataCopyingWrapper(c *minio.Client) { minio.NewSourceInfo(bucketName, "srcObject", nil), minio.NewSourceInfo(bucketName, "srcObject", nil), } - dst4, err := minio.NewDestinationInfo(bucketName, "dstObject-4", nil, map[string]string{"notmyheader": "notmyvalue"}) + dst4, err := minio.NewDestinationInfo(bucketName, "dstObject-4", minio.DestInfoOptions{UserMeta: map[string]string{"notmyheader": "notmyvalue"}}) if err != nil { logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) return @@ -8417,7 +9383,7 @@ func testUserMetadataCopyingWrapper(c *minio.Client) { function = "ComposeObject(destination, sources)" args["source"] = srcs args["destination"] = dst4 - err = c.ComposeObject(context.Background(), dst4, srcs) + _, err = c.ComposeObject(context.Background(), dst4, srcs) if err != nil { logError(testName, function, args, startTime, "", "ComposeObject failed", err) return @@ -8663,8 +9629,8 @@ func testStorageClassMetadataCopyObject() { // Make server side copy of object uploaded in previous step src := minio.NewSourceInfo(bucketName, "srcObjectRRSClass", nil) - dst, err := minio.NewDestinationInfo(bucketName, "srcObjectRRSClassCopy", nil, nil) - if err = c.CopyObject(context.Background(), dst, src); err != nil { + dst, err := minio.NewDestinationInfo(bucketName, "srcObjectRRSClassCopy", minio.DestInfoOptions{}) + if _, err = c.CopyObject(context.Background(), dst, src); err != nil { logError(testName, function, args, startTime, "", "CopyObject failed on RRS", err) } @@ -8690,8 +9656,8 @@ func testStorageClassMetadataCopyObject() { // Make server side copy of object uploaded in previous step src = minio.NewSourceInfo(bucketName, "srcObjectSSClass", nil) - dst, err = minio.NewDestinationInfo(bucketName, "srcObjectSSClassCopy", nil, nil) - if err = c.CopyObject(context.Background(), dst, src); err != nil { + dst, err = minio.NewDestinationInfo(bucketName, "srcObjectSSClassCopy", minio.DestInfoOptions{}) + if _, err = c.CopyObject(context.Background(), dst, src); err != nil { logError(testName, function, args, startTime, "", "CopyObject failed on SS", err) } // Fetch the meta data of copied object @@ -8761,14 +9727,20 @@ func testPutObjectNoLengthV2() { args["size"] = bufSize // Upload an object. - n, err := c.PutObject(context.Background(), bucketName, objectName, reader, -1, minio.PutObjectOptions{}) - + _, err = c.PutObject(context.Background(), bucketName, objectName, reader, -1, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectWithSize failed", err) return } - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Expected upload object size "+string(bufSize)+" got "+string(n), err) + + st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject failed", err) + return + } + + if st.Size != int64(bufSize) { + logError(testName, function, args, startTime, "", "Expected upload object size "+string(bufSize)+" got "+string(st.Size), err) return } @@ -8843,14 +9815,25 @@ func testPutObjectsUnknownV2() { objectName := fmt.Sprintf("%sunique%d", bucketName, i) args["objectName"] = objectName - n, err := c.PutObject(context.Background(), bucketName, objectName, rpipe, -1, minio.PutObjectOptions{}) + ui, err := c.PutObject(context.Background(), bucketName, objectName, rpipe, -1, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectStreaming failed", err) return } - args["size"] = n - if n != int64(4) { - logError(testName, function, args, startTime, "", "Expected upload object size "+string(4)+" got "+string(n), err) + + if ui.Size != 4 { + logError(testName, function, args, startTime, "", "Expected upload object size "+string(4)+" got "+string(ui.Size), nil) + return + } + + st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObjectStreaming failed", err) + return + } + + if st.Size != int64(4) { + logError(testName, function, args, startTime, "", "Expected upload object size "+string(4)+" got "+string(st.Size), err) return } @@ -8915,14 +9898,18 @@ func testPutObject0ByteV2() { args["opts"] = minio.PutObjectOptions{} // Upload an object. - n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader([]byte("")), 0, minio.PutObjectOptions{}) - + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader([]byte("")), 0, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectWithSize failed", err) return } - if n != 0 { - logError(testName, function, args, startTime, "", "Expected upload object size 0 but got "+string(n), err) + st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObjectWithSize failed", err) + return + } + if st.Size != 0 { + logError(testName, function, args, startTime, "", "Expected upload object size 0 but got "+string(st.Size), err) return } @@ -9115,26 +10102,36 @@ func testFunctionalV2() { "objectName": objectName, "contentType": "", } - n, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } - if n != int64(len(buf)) { - logError(testName, function, args, startTime, "", "Expected uploaded object length "+string(len(buf))+" got "+string(n), err) + + st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject failed", err) + return + } + if st.Size != int64(len(buf)) { + logError(testName, function, args, startTime, "", "Expected uploaded object length "+string(len(buf))+" got "+string(st.Size), err) return } objectNameNoLength := objectName + "-nolength" args["objectName"] = objectNameNoLength - n, err = c.PutObject(context.Background(), bucketName, objectNameNoLength, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + _, err = c.PutObject(context.Background(), bucketName, objectNameNoLength, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } - - if n != int64(len(buf)) { - logError(testName, function, args, startTime, "", "Expected uploaded object length "+string(len(buf))+" got "+string(n), err) + st, err = c.StatObject(context.Background(), bucketName, objectNameNoLength, minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject failed", err) + return + } + if st.Size != int64(len(buf)) { + logError(testName, function, args, startTime, "", "Expected uploaded object length "+string(len(buf))+" got "+string(st.Size), err) return } @@ -9151,7 +10148,7 @@ func testFunctionalV2() { "objectName": objectName, "isRecursive": isRecursive, } - for obj := range c.ListObjects(context.Background(), bucketName, objectName, isRecursive, doneCh) { + for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{ForceV1: true, Prefix: objectName, Recursive: isRecursive}) { if obj.Key == objectName { objFound = true break @@ -10131,16 +11128,11 @@ func testListObjects() { } } - testList := func(listFn func(context.Context, string, string, bool, <-chan struct{}) <-chan minio.ObjectInfo, bucket string) { - // Create a done channel to control 'ListObjects' go routine. - doneCh := make(chan struct{}) - // Exit cleanly upon return. - defer close(doneCh) - + testList := func(listFn func(context.Context, string, minio.ListObjectsOptions) <-chan minio.ObjectInfo, bucket string, opts minio.ListObjectsOptions) { var objCursor int // check for object name and storage-class from listing object result - for objInfo := range listFn(context.Background(), bucket, "", true, doneCh) { + for objInfo := range listFn(context.Background(), bucket, opts) { if objInfo.Err != nil { logError(testName, function, args, startTime, "", "ListObjects failed unexpectedly", err) return @@ -10160,8 +11152,8 @@ func testListObjects() { } } - testList(c.ListObjects, bucketName) - testList(c.ListObjectsV2, bucketName) + testList(c.ListObjects, bucketName, minio.ListObjectsOptions{Recursive: true, ForceV1: true}) + testList(c.ListObjects, bucketName, minio.ListObjectsOptions{Recursive: true}) // Delete all objects and buckets if err = cleanupBucket(bucketName, c); err != nil { @@ -10244,7 +11236,7 @@ func testRemoveObjectsWithOptions() { go func() { defer close(objectsCh) // List all objects from a bucket-name with a matching prefix. - for object := range c.ListObjects(context.Background(), bucketName, "", true, nil) { + for object := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{ForceV1: true, Recursive: true}) { if object.Err != nil { log.Fatalln(object.Err) } @@ -10252,7 +11244,7 @@ func testRemoveObjectsWithOptions() { } }() - for rErr := range c.RemoveObjects(context.Background(), bucketName, objectsCh) { + for rErr := range c.RemoveObjects(context.Background(), bucketName, objectsCh, minio.RemoveObjectsOptions{}) { // Error is expected here because Retention is set on the object // and RemoveObjects is called without Bypass Governance if rErr.Err == nil { @@ -10267,7 +11259,7 @@ func testRemoveObjectsWithOptions() { go func() { defer close(objectsCh1) // List all objects from a bucket-name with a matching prefix. - for object := range c.ListObjects(context.Background(), bucketName, "", true, nil) { + for object := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{ForceV1: true, Recursive: true}) { if object.Err != nil { log.Fatalln(object.Err) } @@ -10279,7 +11271,7 @@ func testRemoveObjectsWithOptions() { GovernanceBypass: true, } - for rErr := range c.RemoveObjectsWithOptions(context.Background(), bucketName, objectsCh1, opts1) { + for rErr := range c.RemoveObjects(context.Background(), bucketName, objectsCh1, opts1) { // Error is not expected here because Retention is set on the object // and RemoveObjects is called with Bypass Governance logError(testName, function, args, startTime, "", "Error detected during deletion", rErr.Err) @@ -10287,7 +11279,7 @@ func testRemoveObjectsWithOptions() { } // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { + if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "Cleanup failed", err) return } @@ -10369,6 +11361,14 @@ func main() { testPutObjectWithContentLanguage() testListObjects() testRemoveObjectsWithOptions() + testListObjectVersions() + testStatObjectWithVersioning() + testGetObjectWithVersioning() + testCopyObjectWithVersioning() + testComposeObjectWithVersioning() + testRemoveObjectWithVersioning() + testRemoveObjectsWithVersioning() + testObjectTaggingWithVersioning() // SSE-C tests will only work over TLS connection. if tls { diff --git a/go.sum b/go.sum index 6452387c3..213814107 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -10,6 +12,7 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= @@ -22,11 +25,14 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLD github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= -github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -39,6 +45,7 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= From 72f47b4fb3d850ff52ef600be63bd8518f0ee893 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 9 Jul 2020 08:04:24 -0700 Subject: [PATCH 178/215] re-implement CopyObject/ComposeObject API (#1332) new approach is simpler API style and predicable allows future extensions as well ``` CopyObject(ctx, destOptions, srcOptions) ComposeObject(ctx, destOptions, srcOptions....) ``` --- .github/workflows/go-windows.yml | 2 +- .github/workflows/go.yml | 2 +- api-compose-object.go | 366 +++++++-------- api-compose-object_test.go | 47 +- api-datatypes.go | 11 +- api-list.go | 41 +- api-put-object-copy.go | 78 +--- api-put-object-multipart.go | 3 + api-put-object-streaming.go | 4 +- api-put-object.go | 21 +- core.go | 4 +- docs/API.md | 564 +++++++----------------- examples/s3/composeobject.go | 35 +- examples/s3/copyobject-with-new-tags.go | 43 +- examples/s3/copyobject.go | 41 +- examples/s3/makebucketwithobjectlock.go | 48 -- functional_tests.go | 513 ++++++++++----------- go.sum | 11 +- pkg/s3utils/utils.go | 3 + 19 files changed, 707 insertions(+), 1130 deletions(-) delete mode 100644 examples/s3/makebucketwithobjectlock.go diff --git a/.github/workflows/go-windows.yml b/.github/workflows/go-windows.yml index b0824f320..91d240691 100644 --- a/.github/workflows/go-windows.yml +++ b/.github/workflows/go-windows.yml @@ -40,6 +40,6 @@ jobs: New-Item -ItemType Directory -Path "$env:temp/certs-dir" Copy-Item -Path testcerts\* -Destination "$env:temp/certs-dir" Invoke-WebRequest -Uri https://dl.minio.io/server/minio/release/windows-amd64/minio.exe -OutFile $HOME/minio.exe - Start-Process -NoNewWindow -FilePath "$HOME/minio.exe" -ArgumentList "-S", "$env:temp/certs-dir", "server", "$env:temp/fs" + Start-Process -NoNewWindow -FilePath "$HOME/minio.exe" -ArgumentList "-S", "$env:temp/certs-dir", "server", "$env:temp/fs{1...4}" $env:SSL_CERT_FILE = "$env:temp/certs-dir/public.crt" go run functional_tests.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index efd5ec943..ccc2a235c 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -43,5 +43,5 @@ jobs: chmod +x /tmp/minio mkdir -p /tmp/certs-dir cp testcerts/* /tmp/certs-dir - /tmp/minio server --quiet -S /tmp/certs-dir /tmp/fs & + /tmp/minio server --quiet -S /tmp/certs-dir /tmp/fs{1...4} & make diff --git a/api-compose-object.go b/api-compose-object.go index 66582eb12..15eaf3d91 100644 --- a/api-compose-object.go +++ b/api-compose-object.go @@ -32,15 +32,11 @@ import ( "github.com/minio/minio-go/v7/pkg/s3utils" ) -// DestinationInfo - type with information about the object to be -// created via server-side copy requests, using the Compose API. -type DestinationInfo struct { - bucket, object string - opts DestInfoOptions -} +// CopyDestOptions represents options specified by user for CopyObject/ComposeObject APIs +type CopyDestOptions struct { + Bucket string // points to destination bucket + Object string // points to destination object -// DestInfoOptions represents options specified by user for NewDestinationInfo call -type DestInfoOptions struct { // `Encryption` is the key info for server-side-encryption with customer // provided key. If it is nil, no encryption is performed. Encryption encrypt.ServerSide @@ -53,7 +49,13 @@ type DestInfoOptions struct { // if no user-metadata is provided, it is copied from source // (when there is only once source object in the compose // request) - UserMeta map[string]string + UserMetadata map[string]string + // UserMetadata is only set to destination if ReplaceMetadata is true + // other value is UserMetadata is ignored and we preserve src.UserMetadata + // NOTE: if you set this value to true and now metadata is present + // in UserMetadata your destination object will not have any metadata + // set. + ReplaceMetadata bool // `userTags` is the user defined object tags to be set on destination. // This will be set only if the `replaceTags` field is set to true. @@ -67,12 +69,16 @@ type DestInfoOptions struct { // Object Retention related fields Mode RetentionMode RetainUntilDate time.Time + + Size int64 // Needs to be specified if progress bar is specified. + // Progress of the entire copy operation will be sent here. + Progress io.Reader } // Process custom-metadata to remove a `x-amz-meta-` prefix if // present and validate that keys are distinct (after this // prefix removal). -func filterCustomMeta(userMeta map[string]string) (map[string]string, error) { +func filterCustomMeta(userMeta map[string]string) map[string]string { m := make(map[string]string) for k, v := range userMeta { if strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") { @@ -83,172 +89,114 @@ func filterCustomMeta(userMeta map[string]string) (map[string]string, error) { } m[k] = v } - return m, nil + return m } -// NewDestinationInfo - creates a compose-object/copy-source -// destination info object. -func NewDestinationInfo(bucket, object string, destOpts DestInfoOptions) (d DestinationInfo, err error) { - // Input validation. - if err = s3utils.CheckValidBucketName(bucket); err != nil { - return d, err - } - if err = s3utils.CheckValidObjectName(object); err != nil { - return d, err - } - destOpts.UserMeta, err = filterCustomMeta(destOpts.UserMeta) - if err != nil { - return d, err +// Marshal converts all the CopyDestOptions into their +// equivalent HTTP header representation +func (opts CopyDestOptions) Marshal(header http.Header) { + const replaceDirective = "REPLACE" + if opts.ReplaceTags { + header.Set(amzTaggingHeaderDirective, replaceDirective) + if tags := s3utils.TagEncode(opts.UserTags); tags != "" { + header.Set(amzTaggingHeader, tags) + } } - return DestinationInfo{ - bucket: bucket, - object: object, - opts: destOpts, - }, nil -} -// getUserMetaHeadersMap - construct appropriate key-value pairs to send -// as headers from metadata map to pass into copy-object request. For -// single part copy-object (i.e. non-multipart object), enable the -// withCopyDirectiveHeader to set the `x-amz-metadata-directive` to -// `REPLACE`, so that metadata headers from the source are not copied -// over. -func (d *DestinationInfo) getUserMetaHeadersMap(withCopyDirectiveHeader bool) map[string]string { - if len(d.opts.UserMeta) == 0 { - return nil - } - r := make(map[string]string) - if withCopyDirectiveHeader { - r["x-amz-metadata-directive"] = "REPLACE" - } - for k, v := range d.opts.UserMeta { - if isAmzHeader(k) || isStandardHeader(k) || isStorageClassHeader(k) { - r[k] = v - } else { - r["x-amz-meta-"+k] = v - } + if opts.LegalHold != LegalHoldStatus("") { + header.Set(amzLegalHoldHeader, opts.LegalHold.String()) } - return r -} -// SourceInfo - represents a source object to be copied, using -// server-side copying APIs. -type SourceInfo struct { - bucket, object string - versionID string - start, end int64 - encryption encrypt.ServerSide - // Headers to send with the upload-part-copy request involving - // this source object. - Headers http.Header -} + if opts.Mode != RetentionMode("") && !opts.RetainUntilDate.IsZero() { + header.Set(amzLockMode, opts.Mode.String()) + header.Set(amzLockRetainUntil, opts.RetainUntilDate.Format(time.RFC3339)) + } -// NewSourceInfo - create a compose-object/copy-object source info -// object. -// -// `decryptSSEC` is the decryption key using server-side-encryption -// with customer provided key. It may be nil if the source is not -// encrypted. -func NewSourceInfo(bucket, object string, sse encrypt.ServerSide) SourceInfo { - r := SourceInfo{ - bucket: bucket, - object: object, - start: -1, // range is unspecified by default - encryption: sse, - Headers: make(http.Header), + if opts.Encryption != nil { + opts.Encryption.Marshal(header) } - // Set the source header - r.Headers.Set("x-amz-copy-source", s3utils.EncodePath(bucket+"/"+object)) - return r + if opts.ReplaceMetadata { + header.Set("x-amz-metadata-directive", replaceDirective) + for k, v := range filterCustomMeta(opts.UserMetadata) { + if isAmzHeader(k) || isStandardHeader(k) || isStorageClassHeader(k) { + header.Set(k, v) + } else { + header.Set("x-amz-meta-"+k, v) + } + } + } } -// SetVersionID - Set the version ID of the source object -func (s *SourceInfo) SetVersionID(versionID string) error { - if versionID == "" { - return errInvalidArgument("version ID must be non-empty.") +// toDestinationInfo returns a validated copyOptions object. +func (opts CopyDestOptions) validate() (err error) { + // Input validation. + if err = s3utils.CheckValidBucketName(opts.Bucket); err != nil { + return err + } + if err = s3utils.CheckValidObjectName(opts.Object); err != nil { + return err + } + if opts.Progress != nil && opts.Size < 0 { + return errInvalidArgument("For progress bar effective size needs to be specified") } - s.versionID = versionID - s.Headers.Set("x-amz-copy-source", s3utils.EncodePath(s.bucket+"/"+s.object)+"?versionId="+versionID) return nil } -// SetRange - Set the start and end offset of the source object to be -// copied. If this method is not called, the whole source object is -// copied. -func (s *SourceInfo) SetRange(start, end int64) error { - if start > end || start < 0 { - return errInvalidArgument("start must be non-negative, and start must be at most end.") - } - // Note that 0 <= start <= end - s.start, s.end = start, end - return nil +// CopySrcOptions represents a source object to be copied, using +// server-side copying APIs. +type CopySrcOptions struct { + Bucket, Object string + VersionID string + MatchETag string + NoMatchETag string + MatchModifiedSince time.Time + MatchUnmodifiedSince time.Time + MatchRange bool + Start, End int64 + Encryption encrypt.ServerSide } -// SetMatchETagCond - Set ETag match condition. The object is copied -// only if the etag of the source matches the value given here. -func (s *SourceInfo) SetMatchETagCond(etag string) error { - if etag == "" { - return errInvalidArgument("ETag cannot be empty.") +// Marshal converts all the CopySrcOptions into their +// equivalent HTTP header representation +func (opts CopySrcOptions) Marshal(header http.Header) { + // Set the source header + header.Set("x-amz-copy-source", s3utils.EncodePath(opts.Bucket+"/"+opts.Object)) + if opts.VersionID != "" { + header.Set("x-amz-copy-source", s3utils.EncodePath(opts.Bucket+"/"+opts.Object)+"?versionId="+opts.VersionID) } - s.Headers.Set("x-amz-copy-source-if-match", etag) - return nil -} -// SetMatchETagExceptCond - Set the ETag match exception -// condition. The object is copied only if the etag of the source is -// not the value given here. -func (s *SourceInfo) SetMatchETagExceptCond(etag string) error { - if etag == "" { - return errInvalidArgument("ETag cannot be empty.") + if opts.MatchETag != "" { + header.Set("x-amz-copy-source-if-match", opts.MatchETag) + } + if opts.NoMatchETag != "" { + header.Set("x-amz-copy-source-if-none-match", opts.NoMatchETag) } - s.Headers.Set("x-amz-copy-source-if-none-match", etag) - return nil -} -// SetModifiedSinceCond - Set the modified since condition. -func (s *SourceInfo) SetModifiedSinceCond(modTime time.Time) error { - if modTime.IsZero() { - return errInvalidArgument("Input time cannot be 0.") + if !opts.MatchModifiedSince.IsZero() { + header.Set("x-amz-copy-source-if-modified-since", opts.MatchModifiedSince.Format(http.TimeFormat)) + } + if !opts.MatchUnmodifiedSince.IsZero() { + header.Set("x-amz-copy-source-if-unmodified-since", opts.MatchUnmodifiedSince.Format(http.TimeFormat)) } - s.Headers.Set("x-amz-copy-source-if-modified-since", modTime.Format(http.TimeFormat)) - return nil -} -// SetUnmodifiedSinceCond - Set the unmodified since condition. -func (s *SourceInfo) SetUnmodifiedSinceCond(modTime time.Time) error { - if modTime.IsZero() { - return errInvalidArgument("Input time cannot be 0.") + if opts.Encryption != nil { + encrypt.SSECopy(opts.Encryption).Marshal(header) } - s.Headers.Set("x-amz-copy-source-if-unmodified-since", modTime.Format(http.TimeFormat)) - return nil } -// Helper to fetch size and etag of an object using a StatObject call. -func (s *SourceInfo) getProps(c Client) (size int64, etag string, userMeta map[string]string, err error) { - // Get object info - need size and etag here. Also, decryption - // headers are added to the stat request if given. - var objInfo ObjectInfo - opts := StatObjectOptions{GetObjectOptions{ServerSideEncryption: encrypt.SSE(s.encryption)}} - if s.versionID != "" { - opts.VersionID = s.versionID +func (opts CopySrcOptions) validate() (err error) { + // Input validation. + if err = s3utils.CheckValidBucketName(opts.Bucket); err != nil { + return err } - objInfo, err = c.statObject(context.Background(), s.bucket, s.object, opts) - if err != nil { - err = errInvalidArgument(fmt.Sprintf("Could not stat object - %s/%s: %v", s.bucket, s.object, err)) - } else { - size = objInfo.Size - etag = objInfo.ETag - userMeta = make(map[string]string) - for k, v := range objInfo.Metadata { - if strings.HasPrefix(k, "x-amz-meta-") { - if len(v) > 0 { - userMeta[k] = v[0] - } - } - } + if err = s3utils.CheckValidObjectName(opts.Object); err != nil { + return err } - return + if opts.Start > opts.End || opts.Start < 0 { + return errInvalidArgument("start must be non-negative, and start must be at most end.") + } + return nil } // Low level implementation of CopyObject API, supports only upto 5GiB worth of copy. @@ -384,64 +332,69 @@ func (c Client) uploadPartCopy(ctx context.Context, bucket, object, uploadID str return p, nil } -// ComposeObjectWithProgress - creates an object using server-side copying +// ComposeObject - creates an object using server-side copying // of existing objects. It takes a list of source objects (with optional offsets) // and concatenates them into a new object using only server-side copying // operations. Optionally takes progress reader hook for applications to // look at current progress. -func (c Client) ComposeObjectWithProgress(ctx context.Context, dst DestinationInfo, srcs []SourceInfo, progress io.Reader) (UploadInfo, error) { +func (c Client) ComposeObject(ctx context.Context, dst CopyDestOptions, srcs ...CopySrcOptions) (UploadInfo, error) { if len(srcs) < 1 || len(srcs) > maxPartsCount { return UploadInfo{}, errInvalidArgument("There must be as least one and up to 10000 source objects.") } - srcSizes := make([]int64, len(srcs)) - var totalSize, size, totalParts int64 - var srcUserMeta map[string]string - etags := make([]string, len(srcs)) + + for _, src := range srcs { + if err := src.validate(); err != nil { + return UploadInfo{}, err + } + } + + if err := dst.validate(); err != nil { + return UploadInfo{}, err + } + + srcObjectInfos := make([]ObjectInfo, len(srcs)) + srcObjectSizes := make([]int64, len(srcs)) + var totalSize, totalParts int64 var err error for i, src := range srcs { - size, etags[i], srcUserMeta, err = src.getProps(c) + opts := StatObjectOptions{GetObjectOptions{ServerSideEncryption: encrypt.SSE(src.Encryption), VersionID: src.VersionID}} + srcObjectInfos[i], err = c.statObject(context.Background(), src.Bucket, src.Object, opts) if err != nil { return UploadInfo{}, err } - // Error out if client side encryption is used in this source object when - // more than one source objects are given. - if len(srcs) > 1 && src.Headers.Get("x-amz-meta-x-amz-key") != "" { - return UploadInfo{}, errInvalidArgument( - fmt.Sprintf("Client side encryption is used in source object %s/%s", src.bucket, src.object)) - } - + srcCopySize := srcObjectInfos[i].Size // Check if a segment is specified, and if so, is the // segment within object bounds? - if src.start != -1 { + if src.MatchRange { // Since range is specified, // 0 <= src.start <= src.end // so only invalid case to check is: - if src.end >= size { + if src.End >= srcCopySize || src.Start < 0 { return UploadInfo{}, errInvalidArgument( - fmt.Sprintf("SourceInfo %d has invalid segment-to-copy [%d, %d] (size is %d)", - i, src.start, src.end, size)) + fmt.Sprintf("CopySrcOptions %d has invalid segment-to-copy [%d, %d] (size is %d)", + i, src.Start, src.End, srcCopySize)) } - size = src.end - src.start + 1 + srcCopySize = src.End - src.Start + 1 } // Only the last source may be less than `absMinPartSize` - if size < absMinPartSize && i < len(srcs)-1 { + if srcCopySize < absMinPartSize && i < len(srcs)-1 { return UploadInfo{}, errInvalidArgument( - fmt.Sprintf("SourceInfo %d is too small (%d) and it is not the last part", i, size)) + fmt.Sprintf("CopySrcOptions %d is too small (%d) and it is not the last part", i, srcCopySize)) } // Is data to copy too large? - totalSize += size + totalSize += srcCopySize if totalSize > maxMultipartPutObjectSize { return UploadInfo{}, errInvalidArgument(fmt.Sprintf("Cannot compose an object of size %d (> 5TiB)", totalSize)) } // record source size - srcSizes[i] = size + srcObjectSizes[i] = srcCopySize // calculate parts needed for current source - totalParts += partsRequired(size) + totalParts += partsRequired(srcCopySize) // Do we need more parts than we are allowed? if totalParts > maxPartsCount { return UploadInfo{}, errInvalidArgument(fmt.Sprintf( @@ -452,8 +405,8 @@ func (c Client) ComposeObjectWithProgress(ctx context.Context, dst DestinationIn // Single source object case (i.e. when only one source is // involved, it is being copied wholly and at most 5GiB in // size, emptyfiles are also supported). - if (totalParts == 1 && srcs[0].start == -1 && totalSize <= maxPartSize) || (totalSize == 0) { - return c.CopyObjectWithProgress(ctx, dst, srcs[0], progress) + if (totalParts == 1 && srcs[0].Start == -1 && totalSize <= maxPartSize) || (totalSize == 0) { + return c.CopyObject(ctx, dst, srcs[0]) } // Now, handle multipart-copy cases. @@ -461,9 +414,7 @@ func (c Client) ComposeObjectWithProgress(ctx context.Context, dst DestinationIn // 1. Ensure that the object has not been changed while // we are copying data. for i, src := range srcs { - if src.Headers.Get("x-amz-copy-source-if-match") == "" { - src.SetMatchETagCond(etags[i]) - } + src.MatchETag = srcObjectInfos[i].ETag } // 2. Initiate a new multipart upload. @@ -471,17 +422,28 @@ func (c Client) ComposeObjectWithProgress(ctx context.Context, dst DestinationIn // Set user-metadata on the destination object. If no // user-metadata is specified, and there is only one source, // (only) then metadata from source is copied. - userMeta := dst.getUserMetaHeadersMap(false) - metaMap := userMeta - if len(userMeta) == 0 && len(srcs) == 1 { - metaMap = srcUserMeta + var userMeta map[string]string + if dst.ReplaceMetadata { + userMeta = dst.UserMetadata + } else { + userMeta = srcObjectInfos[0].UserMetadata } - metaHeaders := make(map[string]string) - for k, v := range metaMap { - metaHeaders[k] = v + + var userTags map[string]string + if dst.ReplaceTags { + userTags = dst.UserTags + } else { + userTags = srcObjectInfos[0].UserTags } - uploadID, err := c.newUploadID(ctx, dst.bucket, dst.object, PutObjectOptions{ServerSideEncryption: dst.opts.Encryption, UserMetadata: metaHeaders}) + uploadID, err := c.newUploadID(ctx, dst.Bucket, dst.Object, PutObjectOptions{ + ServerSideEncryption: dst.Encryption, + UserMetadata: userMeta, + UserTags: userTags, + Mode: dst.Mode, + RetainUntilDate: dst.RetainUntilDate, + LegalHold: dst.LegalHold, + }) if err != nil { return UploadInfo{}, err } @@ -490,18 +452,12 @@ func (c Client) ComposeObjectWithProgress(ctx context.Context, dst DestinationIn objParts := []CompletePart{} partIndex := 1 for i, src := range srcs { - h := src.Headers - if src.encryption != nil { - encrypt.SSECopy(src.encryption).Marshal(h) - } - // Add destination encryption headers - if dst.opts.Encryption != nil { - dst.opts.Encryption.Marshal(h) - } + var h = make(http.Header) + src.Marshal(h) // calculate start/end indices of parts after // splitting. - startIdx, endIdx := calculateEvenSplits(srcSizes[i], src) + startIdx, endIdx := calculateEvenSplits(srcObjectSizes[i], src) for j, start := range startIdx { end := endIdx[j] @@ -511,13 +467,13 @@ func (c Client) ComposeObjectWithProgress(ctx context.Context, dst DestinationIn fmt.Sprintf("bytes=%d-%d", start, end)) // make upload-part-copy request - complPart, err := c.uploadPartCopy(ctx, dst.bucket, - dst.object, uploadID, partIndex, h) + complPart, err := c.uploadPartCopy(ctx, dst.Bucket, + dst.Object, uploadID, partIndex, h) if err != nil { return UploadInfo{}, err } - if progress != nil { - io.CopyN(ioutil.Discard, progress, end-start+1) + if dst.Progress != nil { + io.CopyN(ioutil.Discard, dst.Progress, end-start+1) } objParts = append(objParts, complPart) partIndex++ @@ -525,7 +481,7 @@ func (c Client) ComposeObjectWithProgress(ctx context.Context, dst DestinationIn } // 4. Make final complete-multipart request. - uploadInfo, err := c.completeMultipartUpload(ctx, dst.bucket, dst.object, uploadID, + uploadInfo, err := c.completeMultipartUpload(ctx, dst.Bucket, dst.Object, uploadID, completeMultipartUpload{Parts: objParts}) if err != nil { return UploadInfo{}, err @@ -535,14 +491,6 @@ func (c Client) ComposeObjectWithProgress(ctx context.Context, dst DestinationIn return uploadInfo, nil } -// ComposeObject - creates an object using server-side copying of -// existing objects. It takes a list of source objects (with optional -// offsets) and concatenates them into a new object using only -// server-side copying operations. -func (c Client) ComposeObject(ctx context.Context, dst DestinationInfo, srcs []SourceInfo) (UploadInfo, error) { - return c.ComposeObjectWithProgress(ctx, dst, srcs, nil) -} - // partsRequired is maximum parts possible with // max part size of ceiling(maxMultipartPutObjectSize / (maxPartsCount - 1)) func partsRequired(size int64) int64 { @@ -558,7 +506,7 @@ func partsRequired(size int64) int64 { // start and end index slices. Splits happen evenly to be sure that no // part is less than 5MiB, as that could fail the multipart request if // it is not the last part. -func calculateEvenSplits(size int64, src SourceInfo) (startIndex, endIndex []int64) { +func calculateEvenSplits(size int64, src CopySrcOptions) (startIndex, endIndex []int64) { if size == 0 { return } @@ -579,7 +527,7 @@ func calculateEvenSplits(size int64, src SourceInfo) (startIndex, endIndex []int // size = q * k + r (by simple division of size by k, // so that 0 <= r < k) // - start := src.start + start := src.Start if start == -1 { start = 0 } diff --git a/api-compose-object_test.go b/api-compose-object_test.go index ab54af077..fbf8f66a7 100644 --- a/api-compose-object_test.go +++ b/api-compose-object_test.go @@ -17,6 +17,7 @@ package minio import ( + "net/http" "reflect" "strings" "testing" @@ -58,17 +59,17 @@ func TestCalculateEvenSplits(t *testing.T) { testCases := []struct { // input size and source object size int64 - src SourceInfo + src CopySrcOptions // output part-indexes starts, ends []int64 }{ - {0, SourceInfo{start: -1}, nil, nil}, - {1, SourceInfo{start: -1}, []int64{0}, []int64{0}}, - {1, SourceInfo{start: 0}, []int64{0}, []int64{0}}, + {0, CopySrcOptions{Start: -1}, nil, nil}, + {1, CopySrcOptions{Start: -1}, []int64{0}, []int64{0}}, + {1, CopySrcOptions{Start: 0}, []int64{0}, []int64{0}}, - {gb1, SourceInfo{start: -1}, []int64{0, 536870912}, []int64{536870911, 1073741823}}, - {gb5, SourceInfo{start: -1}, + {gb1, CopySrcOptions{Start: -1}, []int64{0, 536870912}, []int64{536870911, 1073741823}}, + {gb5, CopySrcOptions{Start: -1}, []int64{0, 536870912, 1073741824, 1610612736, 2147483648, 2684354560, 3221225472, 3758096384, 4294967296, 4831838208}, []int64{536870911, 1073741823, 1610612735, 2147483647, 2684354559, 3221225471, @@ -76,13 +77,13 @@ func TestCalculateEvenSplits(t *testing.T) { }, // 2 part splits - {gb5p1, SourceInfo{start: -1}, + {gb5p1, CopySrcOptions{Start: -1}, []int64{0, 536870913, 1073741825, 1610612737, 2147483649, 2684354561, 3221225473, 3758096385, 4294967297, 4831838209}, []int64{536870912, 1073741824, 1610612736, 2147483648, 2684354560, 3221225472, 3758096384, 4294967296, 4831838208, 5368709120}, }, - {gb5p1, SourceInfo{start: -1}, + {gb5p1, CopySrcOptions{Start: -1}, []int64{0, 536870913, 1073741825, 1610612737, 2147483649, 2684354561, 3221225473, 3758096385, 4294967297, 4831838209}, []int64{536870912, 1073741824, 1610612736, 2147483648, 2684354560, 3221225472, @@ -90,7 +91,7 @@ func TestCalculateEvenSplits(t *testing.T) { }, // 3 part splits - {gb10p1, SourceInfo{start: -1}, + {gb10p1, CopySrcOptions{Start: -1}, []int64{0, 536870913, 1073741825, 1610612737, 2147483649, 2684354561, 3221225473, 3758096385, 4294967297, 4831838209, 5368709121, 5905580033, 6442450945, 6979321857, 7516192769, 8053063681, @@ -100,7 +101,7 @@ func TestCalculateEvenSplits(t *testing.T) { 5905580032, 6442450944, 6979321856, 7516192768, 8053063680, 8589934592, 9126805504, 9663676416, 10200547328, 10737418240}, }, - {gb10p2, SourceInfo{start: -1}, + {gb10p2, CopySrcOptions{Start: -1}, []int64{0, 536870913, 1073741826, 1610612738, 2147483650, 2684354562, 3221225474, 3758096386, 4294967298, 4831838210, 5368709122, 5905580034, 6442450946, 6979321858, 7516192770, 8053063682, @@ -120,8 +121,7 @@ func TestCalculateEvenSplits(t *testing.T) { } } -func TestGetUserMetaHeadersMap(t *testing.T) { - +func TestDestOptions(t *testing.T) { userMetadata := map[string]string{ "test": "test", "x-amz-acl": "public-read-write", @@ -130,26 +130,27 @@ func TestGetUserMetaHeadersMap(t *testing.T) { "x-amz-grant-write": "test@exo.ch", } - destInfo, _ := NewDestinationInfo("bucket", "object", DestInfoOptions{UserMeta: userMetadata}) - - r := destInfo.getUserMetaHeadersMap(true) + r := make(http.Header) - i := 0 + dst := CopyDestOptions{ + Bucket: "bucket", + Object: "object", + ReplaceMetadata: true, + UserMetadata: userMetadata, + } + dst.Marshal(r) - if _, ok := r["x-amz-metadata-directive"]; !ok { - t.Errorf("Test %d - metadata directive was expected but is missing", i) - i++ + if v := r.Get("x-amz-metadata-directive"); v != "REPLACE" { + t.Errorf("Test - metadata directive was expected but is missing") } for k := range r { if strings.HasSuffix(k, "test") && !strings.HasPrefix(k, "x-amz-meta-") { - t.Errorf("Test %d - meta %q was expected as an x amz meta", i, k) - i++ + t.Errorf("Test meta %q was expected as an x amz meta", k) } if !strings.HasSuffix(k, "test") && strings.HasPrefix(k, "x-amz-meta-") { - t.Errorf("Test %d - an amz/standard/storageClass Header was expected but got an x amz meta data", i) - i++ + t.Errorf("Test an amz/standard/storageClass Header was expected but got an x amz meta data") } } } diff --git a/api-datatypes.go b/api-datatypes.go index d4dbfe7c4..890b60271 100644 --- a/api-datatypes.go +++ b/api-datatypes.go @@ -71,9 +71,14 @@ type Owner struct { // UploadInfo contains information about the // newly uploaded or copied object. type UploadInfo struct { - ETag string - VersionID string - Size int64 + Bucket string + Key string + ETag string + Size int64 + LastModified time.Time + Expiration time.Time + Location string + VersionID string } // ObjectInfo container for object metadata. diff --git a/api-list.go b/api-list.go index a683f8150..784542416 100644 --- a/api-list.go +++ b/api-list.go @@ -58,7 +58,7 @@ func (c Client) ListBuckets(ctx context.Context) ([]BucketInfo, error) { /// Bucket Read Operations. -func (c Client) listObjectsV2(ctx context.Context, bucketName, objectPrefix, marker string, recursive, metadata bool, maxKeys int) <-chan ObjectInfo { +func (c Client) listObjectsV2(ctx context.Context, bucketName, objectPrefix string, recursive, metadata bool, maxKeys int) <-chan ObjectInfo { // Allocate new list objects channel. objectStatCh := make(chan ObjectInfo, 1) // Default listing is delimited at "/" @@ -97,7 +97,7 @@ func (c Client) listObjectsV2(ctx context.Context, bucketName, objectPrefix, mar for { // Get list of objects a maximum of 1000 per request. result, err := c.listObjectsV2Query(ctx, bucketName, objectPrefix, continuationToken, - fetchOwner, metadata, delimiter, maxKeys, marker) + fetchOwner, metadata, delimiter, maxKeys) if err != nil { objectStatCh <- ObjectInfo{ Err: err, @@ -152,9 +152,8 @@ func (c Client) listObjectsV2(ctx context.Context, bucketName, objectPrefix, mar // ?delimiter - A delimiter is a character you use to group keys. // ?prefix - Limits the response to keys that begin with the specified prefix. // ?max-keys - Sets the maximum number of keys returned in the response body. -// ?start-after - Specifies the key to start after when listing objects in a bucket. // ?metadata - Specifies if we want metadata for the objects as part of list operation. -func (c Client) listObjectsV2Query(ctx context.Context, bucketName, objectPrefix, continuationToken string, fetchOwner, metadata bool, delimiter string, maxkeys int, startAfter string) (ListBucketV2Result, error) { +func (c Client) listObjectsV2Query(ctx context.Context, bucketName, objectPrefix, continuationToken string, fetchOwner, metadata bool, delimiter string, maxkeys int) (ListBucketV2Result, error) { // Validate bucket name. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ListBucketV2Result{}, err @@ -198,11 +197,6 @@ func (c Client) listObjectsV2Query(ctx context.Context, bucketName, objectPrefix urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys)) } - // Set start-after - if startAfter != "" { - urlValues.Set("start-after", startAfter) - } - // Execute GET on bucket to list objects. resp, err := c.executeMethod(ctx, "GET", requestMetadata{ bucketName: bucketName, @@ -252,7 +246,7 @@ func (c Client) listObjectsV2Query(ctx context.Context, bucketName, objectPrefix return listBucketResult, nil } -func (c Client) listObjects(ctx context.Context, bucketName, objectPrefix, marker string, recursive bool, maxKeys int) <-chan ObjectInfo { +func (c Client) listObjects(ctx context.Context, bucketName, objectPrefix string, recursive bool, maxKeys int) <-chan ObjectInfo { // Allocate new list objects channel. objectStatCh := make(chan ObjectInfo, 1) // Default listing is delimited at "/" @@ -282,6 +276,7 @@ func (c Client) listObjects(ctx context.Context, bucketName, objectPrefix, marke go func(objectStatCh chan<- ObjectInfo) { defer close(objectStatCh) + marker := "" for { // Get list of objects a maximum of 1000 per request. result, err := c.listObjectsQuery(ctx, bucketName, objectPrefix, marker, delimiter, maxKeys) @@ -331,7 +326,7 @@ func (c Client) listObjects(ctx context.Context, bucketName, objectPrefix, marke return objectStatCh } -func (c Client) listObjectVersions(ctx context.Context, bucketName, prefix, marker string, recursive bool, maxKeys int) <-chan ObjectInfo { +func (c Client) listObjectVersions(ctx context.Context, bucketName, prefix string, recursive bool, maxKeys int) <-chan ObjectInfo { // Allocate new list objects channel. resultCh := make(chan ObjectInfo, 1) // Default listing is delimited at "/" @@ -364,7 +359,7 @@ func (c Client) listObjectVersions(ctx context.Context, bucketName, prefix, mark defer close(resultCh) var ( - keyMarker = marker + keyMarker = "" versionIDMarker = "" ) @@ -619,8 +614,6 @@ func (c Client) listObjectsQuery(ctx context.Context, bucketName, objectPrefix, // ListObjectsOptions holds all options of a list object request type ListObjectsOptions struct { - // Force the deprecated listing V1 - ForceV1 bool // Include objects versions in the listing WithVersions bool // Include objects metadata in the listing @@ -629,10 +622,13 @@ type ListObjectsOptions struct { Prefix string // Ignore '/' delimiter Recursive bool - // Start listing after the specified path - StartAfter string - // The maximum number of returned objects per request + // The maximum number of objects requested per + // batch, advanced use-case not useful for most + // applications MaxKeys int + + // Use the deprecated list objects V1 API + UseV1 bool } // ListObjects returns objects list after evaluating the passed options. @@ -644,21 +640,22 @@ type ListObjectsOptions struct { // func (c Client) ListObjects(ctx context.Context, bucketName string, opts ListObjectsOptions) <-chan ObjectInfo { if opts.WithVersions { - return c.listObjectVersions(ctx, bucketName, opts.Prefix, opts.StartAfter, opts.Recursive, opts.MaxKeys) + return c.listObjectVersions(ctx, bucketName, opts.Prefix, opts.Recursive, opts.MaxKeys) } - if opts.ForceV1 { - return c.listObjects(ctx, bucketName, opts.Prefix, opts.StartAfter, opts.Recursive, opts.MaxKeys) + // Use legacy list objects v1 API + if opts.UseV1 { + return c.listObjects(ctx, bucketName, opts.Prefix, opts.Recursive, opts.MaxKeys) } // Check whether this is snowball region, if yes ListObjectsV2 doesn't work, fallback to listObjectsV1. if location, ok := c.bucketLocCache.Get(bucketName); ok { if location == "snowball" { - return c.listObjects(ctx, bucketName, opts.Prefix, opts.StartAfter, opts.Recursive, opts.MaxKeys) + return c.listObjects(ctx, bucketName, opts.Prefix, opts.Recursive, opts.MaxKeys) } } - return c.listObjectsV2(ctx, bucketName, opts.Prefix, opts.StartAfter, opts.Recursive, opts.WithMetadata, opts.MaxKeys) + return c.listObjectsV2(ctx, bucketName, opts.Prefix, opts.Recursive, opts.WithMetadata, opts.MaxKeys) } // ListIncompleteUploads - List incompletely uploaded multipart objects. diff --git a/api-put-object-copy.go b/api-put-object-copy.go index baf724760..13e17dcf5 100644 --- a/api-put-object-copy.go +++ b/api-put-object-copy.go @@ -22,62 +22,25 @@ import ( "io" "io/ioutil" "net/http" - "time" - - "github.com/minio/minio-go/v7/pkg/encrypt" - "github.com/minio/minio-go/v7/pkg/s3utils" ) // CopyObject - copy a source object into a new object -func (c Client) CopyObject(ctx context.Context, dst DestinationInfo, src SourceInfo) (UploadInfo, error) { - return c.CopyObjectWithProgress(ctx, dst, src, nil) -} - -// CopyObjectWithProgress is like CopyObject with additional progress bar. -func (c Client) CopyObjectWithProgress(ctx context.Context, dst DestinationInfo, src SourceInfo, progress io.Reader) (UploadInfo, error) { - header := make(http.Header) - for k, v := range src.Headers { - header[k] = v - } - - if dst.opts.ReplaceTags && len(dst.opts.UserTags) != 0 { - header.Set(amzTaggingHeaderDirective, "REPLACE") - header.Set(amzTaggingHeader, s3utils.TagEncode(dst.opts.UserTags)) - } - - if dst.opts.LegalHold != LegalHoldStatus("") { - header.Set(amzLegalHoldHeader, dst.opts.LegalHold.String()) - } - - if dst.opts.Mode != RetentionMode("") && !dst.opts.RetainUntilDate.IsZero() { - header.Set(amzLockMode, dst.opts.Mode.String()) - header.Set(amzLockRetainUntil, dst.opts.RetainUntilDate.Format(time.RFC3339)) - } - - var err error - var size int64 - // If progress bar is specified, size should be requested as well initiate a StatObject request. - if progress != nil { - size, _, _, err = src.getProps(c) - if err != nil { - return UploadInfo{}, err - } +func (c Client) CopyObject(ctx context.Context, dst CopyDestOptions, src CopySrcOptions) (UploadInfo, error) { + if err := src.validate(); err != nil { + return UploadInfo{}, err } - if src.encryption != nil { - encrypt.SSECopy(src.encryption).Marshal(header) + if err := dst.validate(); err != nil { + return UploadInfo{}, err } - if dst.opts.Encryption != nil { - dst.opts.Encryption.Marshal(header) - } - for k, v := range dst.getUserMetaHeadersMap(true) { - header.Set(k, v) - } + header := make(http.Header) + dst.Marshal(header) + src.Marshal(header) - resp, err := c.executeMethod(ctx, "PUT", requestMetadata{ - bucketName: dst.bucket, - objectName: dst.object, + resp, err := c.executeMethod(ctx, http.MethodPut, requestMetadata{ + bucketName: dst.Bucket, + objectName: dst.Object, customHeader: header, }) if err != nil { @@ -86,17 +49,24 @@ func (c Client) CopyObjectWithProgress(ctx context.Context, dst DestinationInfo, defer closeResponse(resp) if resp.StatusCode != http.StatusOK { - return UploadInfo{}, httpRespToErrorResponse(resp, dst.bucket, dst.object) + return UploadInfo{}, httpRespToErrorResponse(resp, dst.Bucket, dst.Object) } // Update the progress properly after successful copy. - if progress != nil { - io.CopyN(ioutil.Discard, progress, size) + if dst.Progress != nil { + io.Copy(ioutil.Discard, io.LimitReader(dst.Progress, dst.Size)) + } + + cpObjRes := copyObjectResult{} + if err = xmlDecoder(resp.Body, &cpObjRes); err != nil { + return UploadInfo{}, err } return UploadInfo{ - VersionID: resp.Header.Get("x-amz-version-id"), - Size: size, - ETag: trimEtag(resp.Header.Get("ETag")), + Bucket: dst.Bucket, + Key: dst.Object, + LastModified: cpObjRes.LastModified, + VersionID: resp.Header.Get("x-amz-version-id"), + ETag: trimEtag(resp.Header.Get("ETag")), }, nil } diff --git a/api-put-object-multipart.go b/api-put-object-multipart.go index 978b26505..5d8da15a8 100644 --- a/api-put-object-multipart.go +++ b/api-put-object-multipart.go @@ -369,8 +369,11 @@ func (c Client) completeMultipartUpload(ctx context.Context, bucketName, objectN } return UploadInfo{ + Bucket: completeMultipartUploadResult.Bucket, + Key: completeMultipartUploadResult.Key, ETag: trimEtag(completeMultipartUploadResult.ETag), VersionID: resp.Header.Get("x-amz-version-id"), + Location: completeMultipartUploadResult.Location, }, nil } diff --git a/api-put-object-streaming.go b/api-put-object-streaming.go index 4c8393b62..74e43308a 100644 --- a/api-put-object-streaming.go +++ b/api-put-object-streaming.go @@ -451,8 +451,10 @@ func (c Client) putObjectDo(ctx context.Context, bucketName, objectName string, } return UploadInfo{ + Bucket: bucketName, + Key: objectName, ETag: trimEtag(resp.Header.Get("ETag")), - VersionID: resp.Header.Get("x-amz-version-id"), Size: size, + VersionID: resp.Header.Get("x-amz-version-id"), }, nil } diff --git a/api-put-object.go b/api-put-object.go index 6b7b6e820..dbc830bf9 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -43,8 +43,8 @@ type PutObjectOptions struct { ContentDisposition string ContentLanguage string CacheControl string - Mode *RetentionMode - RetainUntilDate *time.Time + Mode RetentionMode + RetainUntilDate time.Time ServerSideEncryption encrypt.ServerSide NumThreads uint StorageClass string @@ -92,11 +92,11 @@ func (opts PutObjectOptions) Header() (header http.Header) { header.Set("Cache-Control", opts.CacheControl) } - if opts.Mode != nil { + if opts.Mode != "" { header.Set(amzLockMode, opts.Mode.String()) } - if opts.RetainUntilDate != nil { + if !opts.RetainUntilDate.IsZero() { header.Set("X-Amz-Object-Lock-Retain-Until-Date", opts.RetainUntilDate.Format(time.RFC3339)) } @@ -119,15 +119,16 @@ func (opts PutObjectOptions) Header() (header http.Header) { if opts.ReplicationStatus != "" { header.Set(amzBucketReplicationStatus, opts.ReplicationStatus) } + if len(opts.UserTags) != 0 { header.Set(amzTaggingHeader, s3utils.TagEncode(opts.UserTags)) } for k, v := range opts.UserMetadata { - if !isAmzHeader(k) && !isStandardHeader(k) && !isStorageClassHeader(k) { - header.Set("X-Amz-Meta-"+k, v) - } else { + if isAmzHeader(k) || isStandardHeader(k) || isStorageClassHeader(k) { header.Set(k, v) + } else { + header.Set("x-amz-meta-"+k, v) } } return @@ -143,10 +144,8 @@ func (opts PutObjectOptions) validate() (err error) { return errInvalidArgument(v + " unsupported user defined metadata value") } } - if opts.Mode != nil { - if !opts.Mode.IsValid() { - return errInvalidArgument(opts.Mode.String() + " unsupported retention mode") - } + if opts.Mode != "" && !opts.Mode.IsValid() { + return errInvalidArgument(opts.Mode.String() + " unsupported retention mode") } if opts.LegalHold != "" && !opts.LegalHold.IsValid() { return errInvalidArgument(opts.LegalHold.String() + " unsupported legal-hold status") diff --git a/core.go b/core.go index a7f6c7281..f6073225a 100644 --- a/core.go +++ b/core.go @@ -51,8 +51,8 @@ func (c Core) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) // ListObjectsV2 - Lists all the objects at a prefix, similar to ListObjects() but uses // continuationToken instead of marker to support iteration over the results. -func (c Core) ListObjectsV2(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int, startAfter string) (ListBucketV2Result, error) { - return c.listObjectsV2Query(context.Background(), bucketName, objectPrefix, continuationToken, fetchOwner, false, delimiter, maxkeys, startAfter) +func (c Core) ListObjectsV2(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int) (ListBucketV2Result, error) { + return c.listObjectsV2Query(context.Background(), bucketName, objectPrefix, continuationToken, fetchOwner, false, delimiter, maxkeys) } // CopyObject - copies an object from source object to destination object on server side. diff --git a/docs/API.md b/docs/API.md index 61067f23b..b94217f37 100644 --- a/docs/API.md +++ b/docs/API.md @@ -50,29 +50,29 @@ func main() { } ``` -| Bucket operations | Object operations | Encrypted Object operations | Presigned operations | Bucket Policy/Notification Operations | Client custom settings | -| :--- | :--- | :--- | :--- | :--- | :--- | -| [`MakeBucket`](#MakeBucket) | [`GetObject`](#GetObject) | [`GetObject`](#GetObject) | [`PresignedGetObject`](#PresignedGetObject) | [`SetBucketPolicy`](#SetBucketPolicy) | [`SetAppInfo`](#SetAppInfo) | -| [`MakeBucketWithObjectLock`](#MakeBucketWithObjectLock) | [`PutObject`](#PutObject) | [`PutObject`](#PutObject) | [`PresignedPutObject`](#PresignedPutObject) | [`GetBucketPolicy`](#GetBucketPolicy) | [`SetCustomTransport`](#SetCustomTransport) | -| [`ListBuckets`](#ListBuckets) | [`CopyObject`](#CopyObject) | [`CopyObject`](#CopyObject) | [`PresignedPostPolicy`](#PresignedPostPolicy) | [`SetBucketNotification`](#SetBucketNotification) | [`TraceOn`](#TraceOn) | -| [`BucketExists`](#BucketExists) | [`StatObject`](#StatObject) | [`StatObject`](#StatObject) | | [`GetBucketNotification`](#GetBucketNotification) | [`TraceOff`](#TraceOff) | -| [`RemoveBucket`](#RemoveBucket) | [`RemoveObject`](#RemoveObject) | [`FPutObject`](#FPutObject) | | [`RemoveAllBucketNotification`](#RemoveAllBucketNotification) | [`SetS3TransferAccelerate`](#SetS3TransferAccelerate) | -| [`ListObjects`](#ListObjects) | [`RemoveObjects`](#RemoveObjects) | [`FGetObject`](#FGetObject) | | [`ListenBucketNotification`](#ListenBucketNotification) | | -| [`ListObjectsV2`](#ListObjectsV2) | [`RemoveIncompleteUpload`](#RemoveIncompleteUpload) | [`ComposeObject`](#ComposeObjecet) | | [`SetBucketLifecycle`](#SetBucketLifecycle) | | -| [`ListIncompleteUploads`](#ListIncompleteUploads) | [`FPutObject`](#FPutObject) | [`NewSourceInfo`](#NewSourceInfo) | | [`GetBucketLifecycle`](#GetBucketLifecycle) | | -| [`SetBucketTagging`](#SetBucketTagging) | [`FGetObject`](#FGetObject) | [`NewDestinationInfo`](#NewDestinationInfo) | | [`SetObjectLockConfig`](#SetObjectLockConfig) | | -| [`GetBucketTagging`](#GetBucketTagging) | [`ComposeObject`](#ComposeObject) | | | [`GetObjectLockConfig`](#GetObjectLockConfig) | | -| [`DeleteBucketTagging`](#DeleteBucketTagging) | [`NewSourceInfo`](#NewSourceInfo) | | | [`EnableVersioning`](#EnableVersioning) | | -| | [`NewDestinationInfo`](#NewDestinationInfo) | | | [`DisableVersioning`](#DisableVersioning) | | -| | [`PutObjectRetention`](#PutObjectRetention) | | | [`GetBucketEncryption`](#GetBucketEncryption) | | -| | [`GetObjectRetention`](#GetObjectRetention) | | | [`DeleteBucketEncryption`](#DeleteBucketEncryption) | | -| | [`PutObjectLegalHold`](#PutObjectLegalHold) | | | | | -| | [`GetObjectLegalHold`](#GetObjectLegalHold) | | | | | -| | [`SelectObjectContent`](#SelectObjectContent) | | | | | -| | [`PutObjectTagging`](#PutObjectTagging) | | | | | -| | [`GetObjectTagging`](#GetObjectTagging) | | | | | -| | [`RemoveObjectTagging`](#RemoveObjectTagging) | | | | | -| | | | | | | +| Bucket operations | Object operations | Encrypted Object operations | Presigned operations | Bucket Policy/Notification Operations | Client custom settings | +| :--- | :--- | :--- | :--- | :--- | :--- | +| [`MakeBucket`](#MakeBucket) | [`GetObject`](#GetObject) | [`GetObject`](#GetObject) | [`PresignedGetObject`](#PresignedGetObject) | [`SetBucketPolicy`](#SetBucketPolicy) | [`SetAppInfo`](#SetAppInfo) | +| [`PutObject`](#PutObject) | [`PutObject`](#PutObject) | [`PresignedPutObject`](#PresignedPutObject) | [`GetBucketPolicy`](#GetBucketPolicy) | [`SetCustomTransport`](#SetCustomTransport) | +| [`ListBuckets`](#ListBuckets) | [`CopyObject`](#CopyObject) | [`CopyObject`](#CopyObject) | [`PresignedPostPolicy`](#PresignedPostPolicy) | [`SetBucketNotification`](#SetBucketNotification) | [`TraceOn`](#TraceOn) | +| [`BucketExists`](#BucketExists) | [`StatObject`](#StatObject) | [`StatObject`](#StatObject) | | [`GetBucketNotification`](#GetBucketNotification) | [`TraceOff`](#TraceOff) | +| [`RemoveBucket`](#RemoveBucket) | [`RemoveObject`](#RemoveObject) | [`FPutObject`](#FPutObject) | | [`RemoveAllBucketNotification`](#RemoveAllBucketNotification) | [`SetS3TransferAccelerate`](#SetS3TransferAccelerate) | +| [`ListObjects`](#ListObjects) | [`RemoveObjects`](#RemoveObjects) | [`FGetObject`](#FGetObject) | | [`ListenBucketNotification`](#ListenBucketNotification) | | +| | [`RemoveIncompleteUpload`](#RemoveIncompleteUpload) | [`ComposeObject`](#ComposeObjecet) | | [`SetBucketLifecycle`](#SetBucketLifecycle) | | +| [`ListIncompleteUploads`](#ListIncompleteUploads) | [`FPutObject`](#FPutObject) | | | [`GetBucketLifecycle`](#GetBucketLifecycle) | | +| [`SetBucketTagging`](#SetBucketTagging) | [`FGetObject`](#FGetObject) | | | [`SetObjectLockConfig`](#SetObjectLockConfig) | | +| [`GetBucketTagging`](#GetBucketTagging) | [`ComposeObject`](#ComposeObject) | | | [`GetObjectLockConfig`](#GetObjectLockConfig) | | +| [`DeleteBucketTagging`](#DeleteBucketTagging) | | | | [`EnableVersioning`](#EnableVersioning) | | +| | | | | [`DisableVersioning`](#DisableVersioning) | | +| | [`PutObjectRetention`](#PutObjectRetention) | | | [`GetBucketEncryption`](#GetBucketEncryption) | | +| | [`GetObjectRetention`](#GetObjectRetention) | | | [`DeleteBucketEncryption`](#DeleteBucketEncryption) | | +| | [`PutObjectLegalHold`](#PutObjectLegalHold) | | | | | +| | [`GetObjectLegalHold`](#GetObjectLegalHold) | | | | | +| | [`SelectObjectContent`](#SelectObjectContent) | | | | | +| | [`PutObjectTagging`](#PutObjectTagging) | | | | | +| | [`GetObjectTagging`](#GetObjectTagging) | | | | | +| | [`RemoveObjectTagging`](#RemoveObjectTagging) | | | | | +| | | | | | | ## 1. Constructor @@ -82,12 +82,12 @@ Initializes a new client object. __Parameters__ -|Param |Type |Description | -|:---|:---| :---| -|`endpoint` | _string_ |S3 compatible object storage endpoint | -|`accessKeyID` |_string_ |Access key for the object storage | -|`secretAccessKey` | _string_ |Secret key for the object storage | -|`ssl` | _bool_ | If 'true' API requests will be secure (HTTPS), and insecure (HTTP) otherwise | +| Param | Type | Description | +|:------------------|:---------|:-----------------------------------------------------------------------------| +| `endpoint` | _string_ | S3 compatible object storage endpoint | +| `accessKeyID` | _string_ | Access key for the object storage | +| `secretAccessKey` | _string_ | Secret key for the object storage | +| `ssl` | _bool_ | If 'true' API requests will be secure (HTTPS), and insecure (HTTP) otherwise | ### NewWithRegion(endpoint, accessKeyID, secretAccessKey string, ssl bool, region string) (*Client, error) Initializes minio client, with region configured. Unlike New(), NewWithRegion avoids bucket-location lookup operations and it is slightly faster. Use this function when your application deals with a single region. @@ -97,99 +97,66 @@ Initializes minio client with options configured. __Parameters__ -|Param |Type |Description | -|:---|:---| :---| -|`endpoint` | _string_ |S3 compatible object storage endpoint | -|`opts` |_minio.Options_ | Options for constructing a new client| +| Param | Type | Description | +|:-----------|:----------------|:--------------------------------------| +| `endpoint` | _string_ | S3 compatible object storage endpoint | +| `opts` | _minio.Options_ | Options for constructing a new client | __minio.Options__ -|Field | Type | Description | -|:--- |:--- | :--- | -| `opts.Creds` | _*credentials.Credentials_ | Access Credentials| -| `opts.Secure` | _bool_ | If 'true' API requests will be secure (HTTPS), and insecure (HTTP) otherwise | -| `opts.Region` | _string_ | region | -| `opts.BucketLookup` | _BucketLookupType_ | Bucket lookup type can be one of the following values | -| | | _minio.BucketLookupDNS_ | -| | | _minio.BucketLookupPath_ | -| | | _minio.BucketLookupAuto_ | +| Field | Type | Description | +|:--------------------|:---------------------------|:-----------------------------------------------------------------------------| +| `opts.Creds` | _*credentials.Credentials_ | Access Credentials | +| `opts.Secure` | _bool_ | If 'true' API requests will be secure (HTTPS), and insecure (HTTP) otherwise | +| `opts.Region` | _string_ | region | +| `opts.BucketLookup` | _BucketLookupType_ | Bucket lookup type can be one of the following values | +| | | _minio.BucketLookupDNS_ | +| | | _minio.BucketLookupPath_ | +| | | _minio.BucketLookupAuto_ | ## 2. Bucket operations -### MakeBucket(ctx context.Context, bucketName, location string) error +### MakeBucket(ctx context.Context, bucketName string, opts MakeBucketOptions) Creates a new bucket. __Parameters__ -| Param | Type | Description | -|---|---|---| -|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| -|`bucketName` | _string_ | Name of the bucket | -| `location` | _string_ | Region where the bucket is to be created. Default value is us-east-1. Other valid values are listed below. Note: When used with minio server, use the region specified in its config file (defaults to us-east-1).| -| | |us-east-1 | -| | |us-east-2 | -| | |us-west-1 | -| | |us-west-2 | -| | |ca-central-1 | -| | |eu-west-1 | -| | |eu-west-2 | -| | |eu-west-3 | -| | | eu-central-1| -| | | eu-north-1| -| | | ap-east-1| -| | | ap-south-1| -| | | ap-southeast-1| -| | | ap-southeast-2| -| | | ap-northeast-1| -| | | ap-northeast-2| -| | | ap-northeast-3| -| | | me-south-1| -| | | sa-east-1| -| | | us-gov-west-1| -| | | us-gov-east-1| -| | | cn-north-1| -| | | cn-northwest-1| - - -__Example__ - - -```go -err = minioClient.MakeBucket(context.Background(), "mybucket", "us-east-1") -if err != nil { - fmt.Println(err) - return -} -fmt.Println("Successfully created mybucket.") -``` - - -### MakeBucketWithObjectLock(ctx context.Background, bucketName, location string) error -Creates a new bucket with object lock enabled. - -__Parameters__ - -| Param | Type | Description | -|---|---|---| -|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| -|`bucketName` | _string_ | Name of the bucket | -| `location` | _string_ | Region where the bucket is to be created. Default value is us-east-1. Other valid values are listed below. Note: When used with minio server, use the region specified in its config file (defaults to us-east-1).| -| | |us-east-1 | -| | |us-west-1 | -| | |us-west-2 | -| | |eu-west-1 | -| | | eu-central-1| -| | | ap-southeast-1| -| | | ap-northeast-1| -| | | ap-southeast-2| -| | | sa-east-1| +| Param | Type | Description | +|--------------|---------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `ctx` | _context.Context_ | Custom context for timeout/cancellation of the call | +| `bucketName` | _string_ | Name of the bucket | +| `opts` | _minio.MakeBucketOptions_ | Bucket options such as `Region` where the bucket is to be created. Default value is us-east-1. Other valid values are listed below. Note: When used with minio server, use the region specified in its config file (defaults to us-east-1). | +| | | us-east-1 | +| | | us-east-2 | +| | | us-west-1 | +| | | us-west-2 | +| | | ca-central-1 | +| | | eu-west-1 | +| | | eu-west-2 | +| | | eu-west-3 | +| | | eu-central-1 | +| | | eu-north-1 | +| | | ap-east-1 | +| | | ap-south-1 | +| | | ap-southeast-1 | +| | | ap-southeast-2 | +| | | ap-northeast-1 | +| | | ap-northeast-2 | +| | | ap-northeast-3 | +| | | me-south-1 | +| | | sa-east-1 | +| | | us-gov-west-1 | +| | | us-gov-east-1 | +| | | cn-north-1 | +| | | cn-northwest-1 | __Example__ ```go -err = minioClient.MakeBucketWithObjectLock(context.Background(), "mybucket", "us-east-1") +// Create a bucket at region 'us-east-1' with object locking enabled. +err = minioClient.MakeBucket(context.Background(), "mybucket", minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { fmt.Println(err) return @@ -288,19 +255,17 @@ if err != nil { ``` -### ListObjects(ctx context.Context, bucketName, prefix string, recursive bool, doneCh chan struct{}) <-chan ObjectInfo +### ListObjects(ctx context.Context, bucketName string, opts ListObjectsOptions) <-chan ObjectInfo Lists objects in a bucket. __Parameters__ -|Param |Type |Description | -|:---|:---| :---| -|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| -|`bucketName` | _string_ |Name of the bucket | -|`objectPrefix` |_string_ | Prefix of objects to be listed | -|`recursive` | _bool_ |`true` indicates recursive style listing and `false` indicates directory style listing delimited by '/'. | -|`doneCh` | _chan struct{}_ | A message on this channel ends the ListObjects iterator. | +| Param | Type | Description | +|:-------------|:---------------------------|:----------------------------------------------------| +| `ctx` | _context.Context_ | Custom context for timeout/cancellation of the call | +| `bucketName` | _string_ | Name of the bucket | +| `opts` | _minio.ListObjectsOptions_ | Options per to list objects | __Return Value__ @@ -320,14 +285,14 @@ __minio.ObjectInfo__ ```go -// Create a done channel to control 'ListObjects' go routine. -doneCh := make(chan struct{}) +ctx, cancel := context.WithCancel(context.Background()) -// Indicate to our routine to exit cleanly upon return. -defer close(doneCh) +defer cancel() -isRecursive := true -objectCh := minioClient.ListObjects(context.Background(), "mybucket", "myprefix", isRecursive, doneCh) +objectCh := minioClient.ListObjects(ctx, "mybucket", ListObjectOptions{ + Prefix: "myprefix", + Recursive: true, +}) for object := range objectCh { if object.Err != nil { fmt.Println(object.Err) @@ -338,47 +303,6 @@ for object := range objectCh { ``` - -### ListObjectsV2(ctx context.Context, bucketName, prefix string, recursive bool, doneCh chan struct{}) <-chan ObjectInfo -Lists objects in a bucket using the recommended listing API v2 - -__Parameters__ - - -|Param |Type |Description | -|:---|:---| :---| -|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| -|`bucketName` | _string_ |Name of the bucket | -| `objectPrefix` |_string_ | Prefix of objects to be listed | -| `recursive` | _bool_ |`true` indicates recursive style listing and `false` indicates directory style listing delimited by '/'. | -|`doneCh` | _chan struct{}_ | A message on this channel ends the ListObjectsV2 iterator. | - - -__Return Value__ - -|Param |Type |Description | -|:---|:---| :---| -|`objectInfo` | _chan minio.ObjectInfo_ |Read channel for all the objects in the bucket, the object is of the format listed below: | - - -```go -// Create a done channel to control 'ListObjectsV2' go routine. -doneCh := make(chan struct{}) - -// Indicate to our routine to exit cleanly upon return. -defer close(doneCh) - -isRecursive := true -objectCh := minioClient.ListObjectsV2(context.Background(), "mybucket", "myprefix", isRecursive, doneCh) -for object := range objectCh { - if object.Err != nil { - fmt.Println(object.Err) - return - } - fmt.Println(object) -} -``` - ### ListIncompleteUploads(ctx context.Context, bucketName, prefix string, recursive bool) <- chan ObjectMultipartInfo Lists partially uploaded objects in a bucket. @@ -387,12 +311,12 @@ Lists partially uploaded objects in a bucket. __Parameters__ -|Param |Type |Description | -|:---|:---| :---| -|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| -|`bucketName` | _string_ |Name of the bucket | -| `prefix` |_string_ | Prefix of objects that are partially uploaded | -| `recursive` | _bool_ |`true` indicates recursive style listing and `false` indicates directory style listing delimited by '/'. | +| Param | Type | Description | +|:-------------|:------------------|:---------------------------------------------------------------------------------------------------------| +| `ctx` | _context.Context_ | Custom context for timeout/cancellation of the call | +| `bucketName` | _string_ | Name of the bucket | +| `prefix` | _string_ | Prefix of objects that are partially uploaded | +| `recursive` | _bool_ | `true` indicates recursive style listing and `false` indicates directory style listing delimited by '/'. | __Return Value__ @@ -430,11 +354,11 @@ Sets tags to a bucket. __Parameters__ -| Param | Type | Description | -|:-------------|:-------------|:-------------------| -|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| -| `bucketName` | _string_ | Name of the bucket | -| `tags` | _*tags.Tags_ | Bucket tags | +| Param | Type | Description | +|:-------------|:------------------|:----------------------------------------------------| +| `ctx` | _context.Context_ | Custom context for timeout/cancellation of the call | +| `bucketName` | _string_ | Name of the bucket | +| `tags` | _*tags.Tags_ | Bucket tags | __Example__ ```go @@ -653,29 +577,26 @@ API methods PutObjectWithSize, PutObjectWithMetadata, PutObjectStreaming, and Pu -### CopyObject(ctx context.Context, dst DestinationInfo, src SourceInfo) (UploadInfo, error) -Create or replace an object through server-side copying of an existing object. It supports conditional copying, copying a part of an object and server-side encryption of destination and decryption of source. See the `SourceInfo` and `DestinationInfo` types for further details. +### CopyObject(ctx context.Context, dst CopyDestOptions, src CopySrcOptions) (UploadInfo, error) +Create or replace an object through server-side copying of an existing object. It supports conditional copying, copying a part of an object and server-side encryption of destination and decryption of source. See the `CopySrcOptions` and `DestinationInfo` types for further details. To copy multiple source objects into a single destination object see the `ComposeObject` API. __Parameters__ - -|Param |Type |Description | -|:---|:---| :---| -|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| -|`dst` | _minio.DestinationInfo_ |Argument describing the destination object | -|`src` | _minio.SourceInfo_ |Argument describing the source object | +| Param | Type | Description | +|:------|:------------------------|:----------------------------------------------------| +| `ctx` | _context.Context_ | Custom context for timeout/cancellation of the call | +| `dst` | _minio.CopyDestOptions_ | Argument describing the destination object | +| `src` | _minio.CopySrcOptions_ | Argument describing the source object | __minio.UploadInfo__ -| Field | Type | Description | -|:--------------------|:---------|:-------------------------------------------| -| `info.ETag` | _string_ | The ETag of the new object | -| `info.VersionID` | _string_ | The version identifyer of the new object | - - +| Field | Type | Description | +|:-----------------|:---------|:-----------------------------------------| +| `info.ETag` | _string_ | The ETag of the new object | +| `info.VersionID` | _string_ | The version identifyer of the new object | __Example__ @@ -684,13 +605,15 @@ __Example__ ```go // Use-case 1: Simple copy object with no conditions. // Source object -src := minio.NewSourceInfo("my-sourcebucketname", "my-sourceobjectname", nil) +srcOpts := minio.CopySrcOptions{ + Bucket: "my-sourcebucketname", + Object: "my-sourceobjectname", +} // Destination object -dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil) -if err != nil { - fmt.Println(err) - return +dstOpts := minio.CopyDestOptions{ + Bucket: "my-bucketname", + Object: "my-objectname", } // Copy object call @@ -712,25 +635,21 @@ fmt.Println("Successfully copied object:", uploadInfo) // 4. copy only first 1MiB of object. // Source object -src := minio.NewSourceInfo("my-sourcebucketname", "my-sourceobjectname", nil) - -// Set matching ETag condition, copy object which matches the following ETag. -src.SetMatchETagCond("31624deb84149d2f8ef9c385918b653a") - -// Set modified condition, copy object modified since 2014 April 1. -src.SetModifiedSinceCond(time.Date(2014, time.April, 1, 0, 0, 0, 0, time.UTC)) - -// Set unmodified condition, copy object unmodified since 2014 April 23. -src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 23, 0, 0, 0, 0, time.UTC)) +srcOpts := minio.CopySrcOptions{ + Bucket: "my-sourcebucketname", + Object: "my-sourceobjectname", + MatchETag: "31624deb84149d2f8ef9c385918b653a", + MatchModifiedSince: time.Date(2014, time.April, 1, 0, 0, 0, 0, time.UTC), + MatchUnmodifiedSince: time.Date(2014, time.April, 23, 0, 0, 0, 0, time.UTC), + Start: 0, + End: 1024*1024-1, +} -// Set copy-range of only first 1MiB of file. -src.SetRange(0, 1024*1024-1) // Destination object -dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil) -if err != nil { - fmt.Println(err) - return +dstOpts := minio.CopyDestOptions{ + Bucket: "my-bucketname", + Object: "my-objectname", } // Copy object call @@ -745,17 +664,17 @@ fmt.Println("Successfully copied object:", uploadInfo) ``` -### ComposeObject(ctx context.Context, dst minio.DestinationInfo, srcs []minio.SourceInfo) (UploadInfo, error) +### ComposeObject(ctx context.Context, dst minio.CopyDestOptions, srcs ...minio.CopySrcOptions) (UploadInfo, error) Create an object by concatenating a list of source objects using server-side copying. __Parameters__ -|Param |Type |Description | -|:---|:---|:---| -|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| -|`dst` | _minio.DestinationInfo_ |Struct with info about the object to be created. | -|`srcs` | _[]minio.SourceInfo_ |Slice of struct with info about source objects to be concatenated in order. | +| Param | Type | Description | +|:-------|:--------------------------|:----------------------------------------------------------------------------| +| `ctx` | _context.Context_ | Custom context for timeout/cancellation of the call | +| `dst` | _minio.CopyDestOptions_ | Struct with info about the object to be created. | +| `srcs` | _...minio.CopySrcOptions_ | Slice of struct with info about source objects to be concatenated in order. | __minio.UploadInfo__ @@ -766,10 +685,8 @@ __minio.UploadInfo__ | `info.VersionID` | _string_ | The version identifyer of the new object | - __Example__ - ```go // Prepare source decryption key (here we assume same key to // decrypt all source objects.) @@ -777,30 +694,39 @@ sseSrc := encrypt.DefaultPBKDF([]byte("password"), []byte("salt")) // Source objects to concatenate. We also specify decryption // key for each -src1 := minio.NewSourceInfo("bucket1", "object1", sseSrc) -src1.SetMatchETagCond("31624deb84149d2f8ef9c385918b653a") - -src2 := minio.NewSourceInfo("bucket2", "object2", sseSrc) -src2.SetMatchETagCond("f8ef9c385918b653a31624deb84149d2") +src1Opts := minio.CopySrcOptions{ + Bucket: "bucket1", + Object: "object1", + Encryption: sseSrc, + MatchETag: "31624deb84149d2f8ef9c385918b653a", +} -src3 := minio.NewSourceInfo("bucket3", "object3", sseSrc) -src3.SetMatchETagCond("5918b653a31624deb84149d2f8ef9c38") +src2Opts := minio.CopySrcOptions{ + Bucket: "bucket2", + Object: "object2", + Encryption: sseSrc, + MatchETag: "f8ef9c385918b653a31624deb84149d2", +} -// Create slice of sources. -srcs := []minio.SourceInfo{src1, src2, src3} +src3Opts := minio.CopySrcOptions{ + Bucket: "bucket3", + Object: "object3", + Encryption: sseSrc, + MatchETag: "5918b653a31624deb84149d2f8ef9c38", +} // Prepare destination encryption key sseDst := encrypt.DefaultPBKDF([]byte("new-password"), []byte("new-salt")) // Create destination info -dst, err := minio.NewDestinationInfo("bucket", "object", sseDst, nil) -if err != nil { - fmt.Println(err) - return +dstOpts := CopyDestOptions{ + Bucket: "bucket", + Object: "object", + Encryption: sseDst, } // Compose object call by concatenating multiple source files. -uploadInfo, err := minioClient.ComposeObject(context.Background(), dst, srcs) +uploadInfo, err := minioClient.ComposeObject(context.Background(), dst, srcs...) if err != nil { fmt.Println(err) return @@ -809,184 +735,6 @@ if err != nil { fmt.Println("Composed object successfully:", uploadInfo) ``` - -### NewSourceInfo(bucket, object string, decryptSSEC *SSEInfo) SourceInfo -Construct a `SourceInfo` object that can be used as the source for server-side copying operations like `CopyObject` and `ComposeObject`. This object can be used to set copy-conditions on the source. - -__Parameters__ - -| Param | Type | Description | -| :--- | :--- | :--- | -| `bucket` | _string_ | Name of the source bucket | -| `object` | _string_ | Name of the source object | -| `sse` | _*encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v7) | - -__Example__ - -```go -// No decryption parameter. -src := minio.NewSourceInfo("bucket", "object", nil) - -// Destination object -dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil) -if err != nil { - fmt.Println(err) - return -} - -// Copy object call -err = minioClient.CopyObject(dst, src) -if err != nil { - fmt.Println(err) - return -} -``` - -```go -// With decryption parameter. -sseSrc := encrypt.DefaultPBKDF([]byte("password"), []byte("salt")) -src := minio.NewSourceInfo("bucket", "object", sseSrc) - -// Destination object -dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil) -if err != nil { - fmt.Println(err) - return -} - -// Copy object call -_, err = minioClient.CopyObject(dst, src) -if err != nil { - fmt.Println(err) - return -} -``` - - -### NewDestinationInfo(bucket, object string, encryptSSEC *SSEInfo, userMeta map[string]string) (DestinationInfo, error) -Construct a `DestinationInfo` object that can be used as the destination object for server-side copying operations like `CopyObject` and `ComposeObject`. - -__Parameters__ - -| Param | Type | Description | -| :--- | :--- | :--- | -| `bucket` | _string_ | Name of the destination bucket | -| `object` | _string_ | Name of the destination object | -| `sse` | _*encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v7) | | -| `userMeta` | _map[string]string_ | User metadata to be set on the destination. If nil, with only one source, user-metadata is copied from source. | - -__Example__ - -```go -// No encryption parameter. -src := minio.NewSourceInfo("bucket", "object", nil) -dst, err := minio.NewDestinationInfo("bucket", "object", nil, nil) -if err != nil { - fmt.Println(err) - return -} - -// Copy object call -_, err = minioClient.CopyObject(dst, src) -if err != nil { - fmt.Println(err) - return -} -``` - -```go -src := minio.NewSourceInfo("bucket", "object", nil) - -// With encryption parameter. -sseDst := encrypt.DefaultPBKDF([]byte("password"), []byte("salt")) -dst, err := minio.NewDestinationInfo("bucket", "object", sseDst, nil) -if err != nil { - fmt.Println(err) - return -} - -// Copy object call -_, err = minioClient.CopyObject(dst, src) -if err != nil { - fmt.Println(err) - return -} -``` - - -### NewDestinationInfo(bucket, object string, destOpts DestInfoOptions) (DestinationInfo, error) -Construct a `DestinationInfo` object that can be used as the destination object for server-side copying operations like `CopyObject` and `ComposeObject`. - -__Parameters__ - -| Param | Type | Description | -| :--- | :--- | :--- | -| `bucket` | _string_ | Name of the destination bucket | -| `object` | _string_ | Name of the destination object | -| `destOpts` | _minio.DestInfoOptions_ | Pointer to struct that allows user to set optional custom metadata, user tags, and server side encryption parameters. | - -__minio.DestInfoOptions__ - -|Field | Type | Description | -|:--- |:--- | :--- | -| `destOpts.Encryption` | _encrypt.ServerSide_ | Interface provided by encrypt package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v7). | -| `destOpts.UserMetadata` | _map[string]string_ | Map of user meta data to be set on destination object. | -| `destOpts.UserTags` | _map[string]string_ | Map of user object tags to be set on destination object. | -| `destOpts.ReplaceTags` | _bool_ | Replace object tags of the destination object. | -| `destOpts.LegalHold` | _minio.LegalHoldStatus_ | LegalHold(En|Dis)abled. | -| `destOpts.Mode` | _minio.RetentionMode_ | Retention mode to be set on copied object. | -| `destOpts.RetainUntilDate` | _time.Time_ | Time until object retention should be applied on copied object. | - -__Example__ - -```go -// No encryption parameter. -src := minio.NewSourceInfo("bucket", "object", nil) -tags := map[string]string{ - "Tag1": "Value1", - "Tag2": "Value2", -} -dst, err := minio.NewDestinationInfo("bucket", "object", minio.DestInfoOptions{ - UserTags: tags, ReplaceTags: true, -}) -if err != nil { - fmt.Println(err) - return -} - -// Copy object call -_, err = minioClient.CopyObject(context.Background(), dst, src) -if err != nil { - fmt.Println(err) - return -} -``` - -```go -src := minio.NewSourceInfo("bucket", "object", nil) - -// With encryption parameter. -sseDst := encrypt.DefaultPBKDF([]byte("password"), []byte("salt")) -tags := map[string]string{ - "Tag1": "Value1", - "Tag2": "Value2", -} -dst, err := minio.NewDestinationInfo("bucket", "object", minio.DestInfoOptions{ - Encryption: sseDst, UserTags: tags, ReplaceTags: true, -}) -if err != nil { - fmt.Println(err) - return -} - -// Copy object call -_, err = minioClient.CopyObject(context.Background(), dst, src) -if err != nil { - fmt.Println(err) - return -} -``` - ### FPutObject(ctx context.Context, bucketName, objectName, filePath, opts PutObjectOptions) (info UploadInfo, err error) Uploads contents from a file to objectName. @@ -1161,7 +909,7 @@ go func() { opts := minio.RemoveObjectsOptions{ GovernanceBypass: true, } - + for rErr := range minioClient.RemoveObjects(context.Background(), "my-bucketname", objectsCh, opts) { fmt.Println("Error detected during deletion: ", rErr) } diff --git a/examples/s3/composeobject.go b/examples/s3/composeobject.go index 33ee0396c..5798ca34e 100644 --- a/examples/s3/composeobject.go +++ b/examples/s3/composeobject.go @@ -2,7 +2,7 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -50,14 +50,26 @@ func main() { // Source objects to concatenate. We also specify decryption // key for each - src1 := minio.NewSourceInfo("bucket1", "object1", decKey) - src1.SetMatchETagCond("31624deb84149d2f8ef9c385918b653a") + src1 := minio.CopySrcOptions{ + Bucket: "bucket1", + Object: "object1", + Encryption: decKey, + MatchETag: "31624deb84149d2f8ef9c385918b653a", + } - src2 := minio.NewSourceInfo("bucket2", "object2", decKey) - src2.SetMatchETagCond("f8ef9c385918b653a31624deb84149d2") + src2 := minio.CopySrcOptions{ + Bucket: "bucket2", + Object: "object2", + Encryption: decKey, + MatchETag: "f8ef9c385918b653a31624deb84149d2", + } - src3 := minio.NewSourceInfo("bucket3", "object3", decKey) - src3.SetMatchETagCond("5918b653a31624deb84149d2f8ef9c38") + src3 := minio.CopySrcOptions{ + Bucket: "bucket3", + Object: "object3", + Encryption: decKey, + MatchETag: "5918b653a31624deb84149d2f8ef9c38", + } // Create slice of sources. srcs := []minio.SourceInfo{src1, src2, src3} @@ -66,12 +78,13 @@ func main() { encKey, _ := encrypt.NewSSEC([]byte{8, 9, 0}) // Create destination info - dst, err := minio.NewDestinationInfo("bucket", "object", minio.DestInfoOptions{Encryption: encKey}) - if err != nil { - log.Fatalln(err) + dst := minio.CopyDestOptions{ + Bucket: "bucket", + Object: "object", + Encryption: encKey, } - uploadInfo, err := s3Client.ComposeObject(context.Background(), dst, srcs) + uploadInfo, err := s3Client.ComposeObject(context.Background(), dst, srcs...) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/copyobject-with-new-tags.go b/examples/s3/copyobject-with-new-tags.go index 744fce587..2cb3dccc6 100644 --- a/examples/s3/copyobject-with-new-tags.go +++ b/examples/s3/copyobject-with-new-tags.go @@ -45,39 +45,30 @@ func main() { // s3Client.TraceOn(os.Stderr) // Source object - src := minio.NewSourceInfo("my-sourcebucketname", "my-sourceobjectname", nil) - - // All following conditions are allowed and can be combined together. - - // Set modified condition, copy object modified since 2014 April. - src.SetModifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) - - // Set unmodified condition, copy object unmodified since 2014 April. - // src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) - - // Set matching ETag condition, copy object which matches the following ETag. - // src.SetMatchETagCond("31624deb84149d2f8ef9c385918b653a") - - // Set matching ETag except condition, copy object which does not match the following ETag. - // src.SetMatchETagExceptCond("31624deb84149d2f8ef9c385918b653a") - - tags := map[string]string{ - "Tag1": "Value1", - "Tag2": "Value2", + src := minio.CopySrcOptions{ + Bucket: "my-sourcebucketname", + Object: "my-sourceobjectname", + // All following conditions are allowed and can be combined together. + // Set modified condition, copy object modified since 2014 April. + MatchModifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC), } // Destination object - dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", minio.DestInfoOptions{ - UserTags: tags, ReplaceTags: true, - }) - if err != nil { - log.Fatalln(err) + dst := minio.CopyDestOptions{ + Bucket: "my-bucketname", + Object: "my-objectname", + ReplaceTags: true, + UserTags: map[string]string{ + "Tag1": "Value1", + "Tag2": "Value2", + }, } // Initiate copy object. - _, err = s3Client.CopyObject(context.Background(), dst, src) + ui, err := s3Client.CopyObject(context.Background(), dst, src) if err != nil { log.Fatalln(err) } - log.Println("Copied source object /my-sourcebucketname/my-sourceobjectname to destination /my-bucketname/my-objectname Successfully.") + + log.Printf("Copied %s, successfully to %s - UploadInfo %v\n", dst, src, ui) } diff --git a/examples/s3/copyobject.go b/examples/s3/copyobject.go index a235a910d..6499b9870 100644 --- a/examples/s3/copyobject.go +++ b/examples/s3/copyobject.go @@ -2,7 +2,7 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -45,32 +45,31 @@ func main() { // s3Client.TraceOn(os.Stderr) // Source object - src := minio.NewSourceInfo("my-sourcebucketname", "my-sourceobjectname", nil) - - // All following conditions are allowed and can be combined together. - - // Set modified condition, copy object modified since 2014 April. - src.SetModifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) - - // Set unmodified condition, copy object unmodified since 2014 April. - // src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) - - // Set matching ETag condition, copy object which matches the following ETag. - // src.SetMatchETagCond("31624deb84149d2f8ef9c385918b653a") - - // Set matching ETag except condition, copy object which does not match the following ETag. - // src.SetMatchETagExceptCond("31624deb84149d2f8ef9c385918b653a") + src := minio.CopySrcOptions{ + Bucket: "my-sourcebucketname", + Object: "my-sourceobjectname", + // All following conditions are allowed and can be combined together. + // Set modified condition, copy object modified since 2014 April. + MatchModifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC), + // Set unmodified condition, copy object unmodified since 2014 April. + // MatchUnmodifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC), + // Set matching ETag condition, copy object which matches the following ETag. + // MatchETag: "31624deb84149d2f8ef9c385918b653a", + // Set matching ETag copy object which does not match the following ETag. + // NoMatchETag: "31624deb84149d2f8ef9c385918b653a", + } // Destination object - dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", minio.DestInfoOptions{}) - if err != nil { - log.Fatalln(err) + dst := minio.CopyDestOptions{ + Bucket: "my-bucketname", + Object: "my-objectname", } // Initiate copy object. - _, err = s3Client.CopyObject(context.Background(), dst, src) + ui, err = s3Client.CopyObject(context.Background(), dst, src) if err != nil { log.Fatalln(err) } - log.Println("Copied source object /my-sourcebucketname/my-sourceobjectname to destination /my-bucketname/my-objectname Successfully.") + + log.Printf("Copied %s, successfully to %s - UploadInfo %v\n", dst, src, ui) } diff --git a/examples/s3/makebucketwithobjectlock.go b/examples/s3/makebucketwithobjectlock.go deleted file mode 100644 index e1fdcdfd2..000000000 --- a/examples/s3/makebucketwithobjectlock.go +++ /dev/null @@ -1,48 +0,0 @@ -// +build ignore - -/* - * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2019 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "context" - "log" - - minio "github.com/minio/minio-go/v7" -) - -func main() { - // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are - // dummy values, please replace them with original values. - - // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. - // This boolean value is the last argument for New(). - - // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically - // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) - if err != nil { - log.Fatalln(err) - } - - err = s3Client.MakeBucketWithObjectLock(context.Background(), "my-bucketname", "us-east-1") - if err != nil { - log.Fatalln(err) - } - log.Println("Success") -} diff --git a/functional_tests.go b/functional_tests.go index e259146c0..d507e584f 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -194,7 +194,7 @@ func cleanupVersionedBucket(bucketName string, c *minio.Client) error { } } } - for objPartInfo := range c.ListIncompleteUploads(context.Background(), bucketName, "", true, doneCh) { + for objPartInfo := range c.ListIncompleteUploads(context.Background(), bucketName, "", true) { if objPartInfo.Err != nil { return objPartInfo.Err } @@ -986,7 +986,6 @@ func testGetObjectWithVersioning() { } if !bytes.Equal(tmpBuffer.Bytes(), buffers[i]) { - fmt.Println(len(tmpBuffer.Bytes()), len(buffers[i])) logError(testName, function, args, startTime, "", "unexpected content of GetObject()", err) return } @@ -1092,20 +1091,21 @@ func testCopyObjectWithVersioning() { } // Copy Source - src := minio.NewSourceInfo(bucketName, objectName, nil) - src.SetVersionID(infos[0].VersionID) - args["src"] = src + srcOpts := minio.CopySrcOptions{ + Bucket: bucketName, + Object: objectName, + VersionID: infos[0].VersionID, + } + args["src"] = srcOpts - dst, err := minio.NewDestinationInfo(bucketName, objectName+"-copy", minio.DestInfoOptions{}) - args["dst"] = dst - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return + dstOpts := minio.CopyDestOptions{ + Bucket: bucketName, + Object: objectName + "-copy", } + args["dst"] = dstOpts // Perform the Copy - _, err = c.CopyObject(context.Background(), dst, src) - if err != nil { + if _, err = c.CopyObject(context.Background(), dstOpts, srcOpts); err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } @@ -1223,22 +1223,24 @@ func testComposeObjectWithVersioning() { // Source objects to concatenate. We also specify decryption // key for each - src1 := minio.NewSourceInfo(bucketName, objectName, nil) - src1.SetVersionID(results[0].VersionID) - src2 := minio.NewSourceInfo(bucketName, objectName, nil) - src2.SetVersionID(results[1].VersionID) + src1 := minio.CopySrcOptions{ + Bucket: bucketName, + Object: objectName, + VersionID: results[0].VersionID, + } - // Create slice of sources. - srcs := []minio.SourceInfo{src1, src2} + src2 := minio.CopySrcOptions{ + Bucket: bucketName, + Object: objectName, + VersionID: results[1].VersionID, + } - // Create destination info - dst, err := minio.NewDestinationInfo(bucketName, objectName+"-copy", minio.DestInfoOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return + dst := minio.CopyDestOptions{ + Bucket: bucketName, + Object: objectName + "-copy", } - _, err = c.ComposeObject(context.Background(), dst, srcs) + _, err = c.ComposeObject(context.Background(), dst, src1, src2) if err != nil { logError(testName, function, args, startTime, "", "ComposeObject failed", err) return @@ -1535,14 +1537,14 @@ func testObjectTaggingWithVersioning() { tagsV1 := map[string]string{"key1": "val1"} err = c.PutObjectTagging(context.Background(), bucketName, objectName, tagsV1, minio.PutObjectTaggingOptions{VersionID: versions[0].VersionID}) if err != nil { - logError(testName, function, args, startTime, "", "PutObjectTaggingWithOptions (1) failed", err) + logError(testName, function, args, startTime, "", "PutObjectTagging (1) failed", err) return } tagsV2 := map[string]string{"key2": "val2"} err = c.PutObjectTagging(context.Background(), bucketName, objectName, tagsV2, minio.PutObjectTaggingOptions{VersionID: versions[1].VersionID}) if err != nil { - logError(testName, function, args, startTime, "", "PutObjectTaggingWithOptions (2) failed", err) + logError(testName, function, args, startTime, "", "PutObjectTagging (2) failed", err) return } @@ -1560,7 +1562,7 @@ func testObjectTaggingWithVersioning() { gotTagsV1, err := c.GetObjectTagging(context.Background(), bucketName, objectName, minio.GetObjectTaggingOptions{VersionID: versions[0].VersionID}) if err != nil { - logError(testName, function, args, startTime, "", "GetObjectTaggingWithOptions failed", err) + logError(testName, function, args, startTime, "", "GetObjectTagging failed", err) return } @@ -1571,7 +1573,7 @@ func testObjectTaggingWithVersioning() { gotTagsV2, err := c.GetObjectTagging(context.Background(), bucketName, objectName, minio.GetObjectTaggingOptions{}) if err != nil { - logError(testName, function, args, startTime, "", "GetObjectTaggingWithContext failed", err) + logError(testName, function, args, startTime, "", "GetObjectTaggingContext failed", err) return } @@ -1582,14 +1584,14 @@ func testObjectTaggingWithVersioning() { err = c.RemoveObjectTagging(context.Background(), bucketName, objectName, minio.RemoveObjectTaggingOptions{VersionID: versions[0].VersionID}) if err != nil { - logError(testName, function, args, startTime, "", "PutObjectTaggingWithOptions (2) failed", err) + logError(testName, function, args, startTime, "", "PutObjectTagging (2) failed", err) return } emptyTags, err := c.GetObjectTagging(context.Background(), bucketName, objectName, minio.GetObjectTaggingOptions{VersionID: versions[0].VersionID}) if err != nil { - logError(testName, function, args, startTime, "", "GetObjectTaggingWithOptions failed", err) + logError(testName, function, args, startTime, "", "GetObjectTagging failed", err) return } @@ -2089,7 +2091,7 @@ func testGetObjectClosedTwice() { } // Test RemoveObjects request where context cancels after timeout -func testRemoveObjectsWithContext() { +func testRemoveObjectsContext() { // Initialize logging params. startTime := time.Now() testName := getFuncName() @@ -2554,7 +2556,7 @@ func testFPutObject() { } // Tests FPutObject request when context cancels after timeout -func testFPutObjectWithContext() { +func testFPutObjectContext() { // initialize logging params startTime := time.Now() testName := getFuncName() @@ -2602,7 +2604,7 @@ func testFPutObjectWithContext() { var fName = getMintDataDirFilePath("datafile-1-MB") if fName == "" { // Make a temp file with 1 MiB bytes of data. - file, err := ioutil.TempFile(os.TempDir(), "FPutObjectWithContextTest") + file, err := ioutil.TempFile(os.TempDir(), "FPutObjectContextTest") if err != nil { logError(testName, function, args, startTime, "", "TempFile creation failed", err) return @@ -2623,7 +2625,7 @@ func testFPutObjectWithContext() { } // Set base object name - objectName := bucketName + "FPutObjectWithContext" + objectName := bucketName + "FPutObjectContext" args["objectName"] = objectName ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) args["ctx"] = ctx @@ -2661,11 +2663,11 @@ func testFPutObjectWithContext() { } // Tests FPutObject request when context cancels after timeout -func testFPutObjectWithContextV2() { +func testFPutObjectContextV2() { // initialize logging params startTime := time.Now() testName := getFuncName() - function := "FPutObjectWithContext(ctx, bucketName, objectName, fileName, opts)" + function := "FPutObjectContext(ctx, bucketName, objectName, fileName, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", @@ -2708,7 +2710,7 @@ func testFPutObjectWithContextV2() { var fName = getMintDataDirFilePath("datafile-1-MB") if fName == "" { // Make a temp file with 1 MiB bytes of data. - file, err := ioutil.TempFile(os.TempDir(), "FPutObjectWithContextTest") + file, err := ioutil.TempFile(os.TempDir(), "FPutObjectContextTest") if err != nil { logError(testName, function, args, startTime, "", "Temp file creation failed", err) return @@ -2730,7 +2732,7 @@ func testFPutObjectWithContextV2() { } // Set base object name - objectName := bucketName + "FPutObjectWithContext" + objectName := bucketName + "FPutObjectContext" args["objectName"] = objectName ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) @@ -2769,7 +2771,7 @@ func testFPutObjectWithContextV2() { } // Test validates putObject with context to see if request cancellation is honored. -func testPutObjectWithContext() { +func testPutObjectContext() { // initialize logging params startTime := time.Now() testName := getFuncName() @@ -3607,63 +3609,26 @@ func testCopyObject() { } // Copy Source - src := minio.NewSourceInfo(bucketName, objectName, nil) - args["src"] = src - - // Set copy conditions. - - // All invalid conditions first. - err = src.SetModifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) - if err == nil { - logError(testName, function, args, startTime, "", "SetModifiedSinceCond did not fail for invalid conditions", err) - return - } - err = src.SetUnmodifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) - if err == nil { - logError(testName, function, args, startTime, "", "SetUnmodifiedSinceCond did not fail for invalid conditions", err) - return - } - err = src.SetMatchETagCond("") - if err == nil { - logError(testName, function, args, startTime, "", "SetMatchETagCond did not fail for invalid conditions", err) - return - } - err = src.SetMatchETagExceptCond("") - if err == nil { - logError(testName, function, args, startTime, "", "SetMatchETagExceptCond did not fail for invalid conditions", err) - return - } - - err = src.SetModifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) - if err != nil { - logError(testName, function, args, startTime, "", "SetModifiedSinceCond failed", err) - return - } - err = src.SetMatchETagCond(objInfo.ETag) - if err != nil { - logError(testName, function, args, startTime, "", "SetMatchETagCond failed", err) - return + src := minio.CopySrcOptions{ + Bucket: bucketName, + Object: objectName, + // Set copy conditions. + MatchETag: objInfo.ETag, + MatchModifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC), } + args["src"] = src - dst, err := minio.NewDestinationInfo(bucketName+"-copy", objectName+"-copy", minio.DestInfoOptions{}) - args["dst"] = dst - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return + dst := minio.CopyDestOptions{ + Bucket: bucketName + "-copy", + Object: objectName + "-copy", } // Perform the Copy - ui, err := c.CopyObject(context.Background(), dst, src) - if err != nil { + if _, err = c.CopyObject(context.Background(), dst, src); err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } - if ui.Size != 0 { - logError(testName, function, args, startTime, "", "CopyObject returned unexpeced size", nil) - return - } - // Source object r, err = c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { @@ -3677,6 +3642,7 @@ func testCopyObject() { logError(testName, function, args, startTime, "", "GetObject failed", err) return } + // Check the various fields of source object against destination object. objInfo, err = r.Stat() if err != nil { @@ -3698,16 +3664,11 @@ func testCopyObject() { readerCopy.Close() // CopyObject again but with wrong conditions - src = minio.NewSourceInfo(bucketName, objectName, nil) - err = src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) - if err != nil { - logError(testName, function, args, startTime, "", "SetUnmodifiedSinceCond failed", err) - return - } - err = src.SetMatchETagExceptCond(objInfo.ETag) - if err != nil { - logError(testName, function, args, startTime, "", "SetMatchETagExceptCond failed", err) - return + src = minio.CopySrcOptions{ + Bucket: bucketName, + Object: objectName, + MatchUnmodifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC), + NoMatchETag: objInfo.ETag, } // Perform the Copy which should fail @@ -3717,21 +3678,21 @@ func testCopyObject() { return } - // Perform the Copy which should update only metadata. - src = minio.NewSourceInfo(bucketName, objectName, nil) - dst, err = minio.NewDestinationInfo(bucketName, objectName, - minio.DestInfoOptions{ - UserMeta: map[string]string{ - "Copy": "should be same", - }, + src = minio.CopySrcOptions{ + Bucket: bucketName, + Object: objectName, + } + + dst = minio.CopyDestOptions{ + Bucket: bucketName, + Object: objectName, + ReplaceMetadata: true, + UserMetadata: map[string]string{ + "Copy": "should be same", }, - ) + } args["dst"] = dst args["src"] = src - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return - } _, err = c.CopyObject(context.Background(), dst, src) if err != nil { @@ -5340,7 +5301,7 @@ func testFunctional() { "isRecursive": isRecursive, } - for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{ForceV1: true, Prefix: objectName, Recursive: true}) { + for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{UseV1: true, Prefix: objectName, Recursive: true}) { if obj.Key == objectName { objFound = true break @@ -6735,50 +6696,20 @@ func testCopyObjectV2() { r.Close() // Copy Source - src := minio.NewSourceInfo(bucketName, objectName, nil) + src := minio.CopySrcOptions{ + Bucket: bucketName, + Object: objectName, + MatchModifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC), + MatchETag: objInfo.ETag, + } args["source"] = src // Set copy conditions. - - // All invalid conditions first. - err = src.SetModifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) - if err == nil { - logError(testName, function, args, startTime, "", "SetModifiedSinceCond did not fail for invalid conditions", err) - return - } - err = src.SetUnmodifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) - if err == nil { - logError(testName, function, args, startTime, "", "SetUnmodifiedSinceCond did not fail for invalid conditions", err) - return - } - err = src.SetMatchETagCond("") - if err == nil { - logError(testName, function, args, startTime, "", "SetMatchETagCond did not fail for invalid conditions", err) - return - } - err = src.SetMatchETagExceptCond("") - if err == nil { - logError(testName, function, args, startTime, "", "SetMatchETagExceptCond did not fail for invalid conditions", err) - return - } - - err = src.SetModifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) - if err != nil { - logError(testName, function, args, startTime, "", "SetModifiedSinceCond failed", err) - return - } - err = src.SetMatchETagCond(objInfo.ETag) - if err != nil { - logError(testName, function, args, startTime, "", "SetMatchETagCond failed", err) - return + dst := minio.CopyDestOptions{ + Bucket: bucketName + "-copy", + Object: objectName + "-copy", } - - dst, err := minio.NewDestinationInfo(bucketName+"-copy", objectName+"-copy", minio.DestInfoOptions{}) args["destination"] = dst - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return - } // Perform the Copy _, err = c.CopyObject(context.Background(), dst, src) @@ -6820,16 +6751,11 @@ func testCopyObjectV2() { readerCopy.Close() // CopyObject again but with wrong conditions - src = minio.NewSourceInfo(bucketName, objectName, nil) - err = src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) - if err != nil { - logError(testName, function, args, startTime, "", "SetUnmodifiedSinceCond failed", err) - return - } - err = src.SetMatchETagExceptCond(objInfo.ETag) - if err != nil { - logError(testName, function, args, startTime, "", "SetMatchETagExceptCond failed", err) - return + src = minio.CopySrcOptions{ + Bucket: bucketName, + Object: objectName, + MatchUnmodifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC), + NoMatchETag: objInfo.ETag, } // Perform the Copy which should fail @@ -6871,19 +6797,18 @@ func testComposeObjectErrorCasesWrapper(c *minio.Client) { // Test that more than 10K source objects cannot be // concatenated. - srcArr := [10001]minio.SourceInfo{} + srcArr := [10001]minio.CopySrcOptions{} srcSlice := srcArr[:] - dst, err := minio.NewDestinationInfo(bucketName, "object", minio.DestInfoOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return + dst := minio.CopyDestOptions{ + Bucket: bucketName, + Object: "object", } args["destination"] = dst // Just explain about srcArr in args["sourceList"] // to stop having 10,001 null headers logged args["sourceList"] = "source array of 10,001 elements" - if _, err := c.ComposeObject(context.Background(), dst, srcSlice); err == nil { + if _, err := c.ComposeObject(context.Background(), dst, srcSlice...); err == nil { logError(testName, function, args, startTime, "", "Expected error in ComposeObject", err) return } else if err.Error() != "There must be as least one and up to 10000 source objects." { @@ -6903,14 +6828,16 @@ func testComposeObjectErrorCasesWrapper(c *minio.Client) { } // 2. Set invalid range spec on the object (going beyond // object size) - badSrc := minio.NewSourceInfo(bucketName, "badObject", nil) - err = badSrc.SetRange(1, badSrcSize) - if err != nil { - logError(testName, function, args, startTime, "", "Setting NewSourceInfo failed", err) - return + badSrc := minio.CopySrcOptions{ + Bucket: bucketName, + Object: "badObject", + MatchRange: true, + Start: 1, + End: badSrcSize, } + // 3. ComposeObject call should fail. - if _, err := c.ComposeObject(context.Background(), dst, []minio.SourceInfo{badSrc}); err == nil { + if _, err := c.ComposeObject(context.Background(), dst, badSrc); err == nil { logError(testName, function, args, startTime, "", "ComposeObject expected to fail", err) return } else if !strings.Contains(err.Error(), "has invalid segment-to-copy") { @@ -6979,26 +6906,26 @@ func testComposeMultipleSources(c *minio.Client) { } // We will append 10 copies of the object. - srcs := []minio.SourceInfo{} + srcs := []minio.CopySrcOptions{} for i := 0; i < 10; i++ { - srcs = append(srcs, minio.NewSourceInfo(bucketName, "srcObject", nil)) + srcs = append(srcs, minio.CopySrcOptions{ + Bucket: bucketName, + Object: "srcObject", + }) } + // make the last part very small - err = srcs[9].SetRange(0, 0) - if err != nil { - logError(testName, function, args, startTime, "", "SetRange failed", err) - return - } + srcs[9].MatchRange = true + args["sourceList"] = srcs - dst, err := minio.NewDestinationInfo(bucketName, "dstObject", minio.DestInfoOptions{}) + dst := minio.CopyDestOptions{ + Bucket: bucketName, + Object: "dstObject", + } args["destination"] = dst - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return - } - ui, err := c.ComposeObject(context.Background(), dst, srcs) + ui, err := c.ComposeObject(context.Background(), dst, srcs...) if err != nil { logError(testName, function, args, startTime, "", "ComposeObject failed", err) return @@ -7092,33 +7019,40 @@ func testEncryptedEmptyObject() { } // 2. Test CopyObject for an empty object - dstInfo, err := minio.NewDestinationInfo(bucketName, "new-object", minio.DestInfoOptions{Encryption: sse}) - if err != nil { - args["objectName"] = "new-object" - function = "NewDestinationInfo(bucketName, objectName, sse, userMetadata)" - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return + src := minio.CopySrcOptions{ + Bucket: bucketName, + Object: "object", + Encryption: sse, + } + + dst := minio.CopyDestOptions{ + Bucket: bucketName, + Object: "new-object", + Encryption: sse, } - srcInfo := minio.NewSourceInfo(bucketName, "object", sse) - if _, err = c.CopyObject(context.Background(), dstInfo, srcInfo); err != nil { - function = "CopyObject(dstInfo, srcInfo)" + + if _, err = c.CopyObject(context.Background(), dst, src); err != nil { + function = "CopyObject(dst, src)" logError(testName, function, map[string]interface{}{}, startTime, "", "CopyObject failed", err) return } // 3. Test Key rotation newSSE := encrypt.DefaultPBKDF([]byte("Don't Panic"), []byte(bucketName+"new-object")) - dstInfo, err = minio.NewDestinationInfo(bucketName, "new-object", minio.DestInfoOptions{Encryption: newSSE}) - if err != nil { - args["objectName"] = "new-object" - function = "NewDestinationInfo(bucketName, objectName, encryptSSEC, userMetadata)" - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return + src = minio.CopySrcOptions{ + Bucket: bucketName, + Object: "new-object", + Encryption: sse, } - srcInfo = minio.NewSourceInfo(bucketName, "new-object", sse) - if _, err = c.CopyObject(context.Background(), dstInfo, srcInfo); err != nil { - function = "CopyObject(dstInfo, srcInfo)" + dst = minio.CopyDestOptions{ + Bucket: bucketName, + Object: "new-object", + Encryption: newSSE, + } + + if _, err = c.CopyObject(context.Background(), dst, src); err != nil { + function = "CopyObject(dst, src)" logError(testName, function, map[string]interface{}{}, startTime, "", "CopyObject with key rotation failed", err) return } @@ -7181,12 +7115,17 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, } // 2. copy object and change encryption key - src := minio.NewSourceInfo(bucketName, "srcObject", srcEncryption) + src := minio.CopySrcOptions{ + Bucket: bucketName, + Object: "srcObject", + Encryption: srcEncryption, + } args["source"] = src - dst, err := minio.NewDestinationInfo(bucketName, "dstObject", minio.DestInfoOptions{Encryption: sseDst}) - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return + + dst := minio.CopyDestOptions{ + Bucket: bucketName, + Object: "dstObject", + Encryption: sseDst, } args["destination"] = dst @@ -7227,10 +7166,10 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, newSSE = encrypt.NewSSE() } if newSSE != nil { - dst, err = minio.NewDestinationInfo(bucketName, "srcObject", minio.DestInfoOptions{Encryption: newSSE}) - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return + dst = minio.CopyDestOptions{ + Bucket: bucketName, + Object: "srcObject", + Encryption: newSSE, } args["destination"] = dst @@ -7257,15 +7196,19 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, return } reader.Close() + // Test in-place decryption. - dst, err = minio.NewDestinationInfo(bucketName, "srcObject", minio.DestInfoOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return + dst = minio.CopyDestOptions{ + Bucket: bucketName, + Object: "srcObject", } args["destination"] = dst - src = minio.NewSourceInfo(bucketName, "srcObject", newSSE) + src = minio.CopySrcOptions{ + Bucket: bucketName, + Object: "srcObject", + Encryption: newSSE, + } args["source"] = src _, err = c.CopyObject(context.Background(), dst, src) if err != nil { @@ -7614,12 +7557,16 @@ func testDecryptedCopyObject() { return } - src := minio.NewSourceInfo(bucketName, objectName, encrypt.SSECopy(encryption)) + src := minio.CopySrcOptions{ + Bucket: bucketName, + Object: objectName, + Encryption: encrypt.SSECopy(encryption), + } args["source"] = src - dst, err := minio.NewDestinationInfo(bucketName, "decrypted-"+objectName, minio.DestInfoOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return + + dst := minio.CopyDestOptions{ + Bucket: bucketName, + Object: "decrypted-" + objectName, } args["destination"] = dst @@ -9267,9 +9214,7 @@ func testUserMetadataCopyingWrapper(c *minio.Client) { h = make(http.Header) for k, vs := range objInfo.Metadata { if strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") { - for _, v := range vs { - h.Add(k, v) - } + h.Add(k, vs[0]) } } return h @@ -9294,12 +9239,17 @@ func testUserMetadataCopyingWrapper(c *minio.Client) { } // 2. create source - src := minio.NewSourceInfo(bucketName, "srcObject", nil) + src := minio.CopySrcOptions{ + Bucket: bucketName, + Object: "srcObject", + } + // 2.1 create destination with metadata set - dst1, err := minio.NewDestinationInfo(bucketName, "dstObject-1", minio.DestInfoOptions{UserMeta: map[string]string{"notmyheader": "notmyvalue"}}) - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return + dst1 := minio.CopyDestOptions{ + Bucket: bucketName, + Object: "dstObject-1", + UserMetadata: map[string]string{"notmyheader": "notmyvalue"}, + ReplaceMetadata: true, } // 3. Check that copying to an object with metadata set resets @@ -9320,12 +9270,10 @@ func testUserMetadataCopyingWrapper(c *minio.Client) { } // 4. create destination with no metadata set and same source - dst2, err := minio.NewDestinationInfo(bucketName, "dstObject-2", minio.DestInfoOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return + dst2 := minio.CopyDestOptions{ + Bucket: bucketName, + Object: "dstObject-2", } - src = minio.NewSourceInfo(bucketName, "srcObject", nil) // 5. Check that copying to an object with no metadata set, // copies metadata. @@ -9344,20 +9292,16 @@ func testUserMetadataCopyingWrapper(c *minio.Client) { } // 6. Compose a pair of sources. - srcs := []minio.SourceInfo{ - minio.NewSourceInfo(bucketName, "srcObject", nil), - minio.NewSourceInfo(bucketName, "srcObject", nil), - } - dst3, err := minio.NewDestinationInfo(bucketName, "dstObject-3", minio.DestInfoOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return + dst3 := minio.CopyDestOptions{ + Bucket: bucketName, + Object: "dstObject-3", + ReplaceMetadata: true, } function = "ComposeObject(destination, sources)" - args["source"] = srcs + args["source"] = []minio.CopySrcOptions{src, src} args["destination"] = dst3 - _, err = c.ComposeObject(context.Background(), dst3, srcs) + _, err = c.ComposeObject(context.Background(), dst3, src, src) if err != nil { logError(testName, function, args, startTime, "", "ComposeObject failed", err) return @@ -9370,20 +9314,17 @@ func testUserMetadataCopyingWrapper(c *minio.Client) { } // 7. Compose a pair of sources with dest user metadata set. - srcs = []minio.SourceInfo{ - minio.NewSourceInfo(bucketName, "srcObject", nil), - minio.NewSourceInfo(bucketName, "srcObject", nil), - } - dst4, err := minio.NewDestinationInfo(bucketName, "dstObject-4", minio.DestInfoOptions{UserMeta: map[string]string{"notmyheader": "notmyvalue"}}) - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return + dst4 := minio.CopyDestOptions{ + Bucket: bucketName, + Object: "dstObject-4", + UserMetadata: map[string]string{"notmyheader": "notmyvalue"}, + ReplaceMetadata: true, } function = "ComposeObject(destination, sources)" - args["source"] = srcs + args["source"] = []minio.CopySrcOptions{src, src} args["destination"] = dst4 - _, err = c.ComposeObject(context.Background(), dst4, srcs) + _, err = c.ComposeObject(context.Background(), dst4, src, src) if err != nil { logError(testName, function, args, startTime, "", "ComposeObject failed", err) return @@ -9628,8 +9569,14 @@ func testStorageClassMetadataCopyObject() { } // Make server side copy of object uploaded in previous step - src := minio.NewSourceInfo(bucketName, "srcObjectRRSClass", nil) - dst, err := minio.NewDestinationInfo(bucketName, "srcObjectRRSClassCopy", minio.DestInfoOptions{}) + src := minio.CopySrcOptions{ + Bucket: bucketName, + Object: "srcObjectRRSClass", + } + dst := minio.CopyDestOptions{ + Bucket: bucketName, + Object: "srcObjectRRSClassCopy", + } if _, err = c.CopyObject(context.Background(), dst, src); err != nil { logError(testName, function, args, startTime, "", "CopyObject failed on RRS", err) } @@ -9655,8 +9602,14 @@ func testStorageClassMetadataCopyObject() { } // Make server side copy of object uploaded in previous step - src = minio.NewSourceInfo(bucketName, "srcObjectSSClass", nil) - dst, err = minio.NewDestinationInfo(bucketName, "srcObjectSSClassCopy", minio.DestInfoOptions{}) + src = minio.CopySrcOptions{ + Bucket: bucketName, + Object: "srcObjectSSClass", + } + dst = minio.CopyDestOptions{ + Bucket: bucketName, + Object: "srcObjectSSClassCopy", + } if _, err = c.CopyObject(context.Background(), dst, src); err != nil { logError(testName, function, args, startTime, "", "CopyObject failed on SS", err) } @@ -10148,7 +10101,7 @@ func testFunctionalV2() { "objectName": objectName, "isRecursive": isRecursive, } - for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{ForceV1: true, Prefix: objectName, Recursive: isRecursive}) { + for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{UseV1: true, Prefix: objectName, Recursive: isRecursive}) { if obj.Key == objectName { objFound = true break @@ -10414,7 +10367,7 @@ func testFunctionalV2() { } // Test get object with GetObject with context -func testGetObjectWithContext() { +func testGetObjectContext() { // initialize logging params startTime := time.Now() testName := getFuncName() @@ -10521,7 +10474,7 @@ func testGetObjectWithContext() { } // Test get object with FGetObject with a user provided context -func testFGetObjectWithContext() { +func testFGetObjectContext() { // initialize logging params startTime := time.Now() testName := getFuncName() @@ -10613,7 +10566,7 @@ func testFGetObjectWithContext() { } // Test get object ACLs with GetObjectACL with custom provided context -func testGetObjectACLWithContext() { +func testGetObjectACLContext() { // initialize logging params startTime := time.Now() testName := getFuncName() @@ -10783,7 +10736,7 @@ func testGetObjectACLWithContext() { } // Test validates putObject with context to see if request cancellation is honored for V2. -func testPutObjectWithContextV2() { +func testPutObjectContextV2() { // initialize logging params startTime := time.Now() testName := getFuncName() @@ -10864,7 +10817,7 @@ func testPutObjectWithContextV2() { } // Test get object with GetObject with custom context -func testGetObjectWithContextV2() { +func testGetObjectContextV2() { // initialize logging params startTime := time.Now() testName := getFuncName() @@ -10969,7 +10922,7 @@ func testGetObjectWithContextV2() { } // Test get object with FGetObject with custom context -func testFGetObjectWithContextV2() { +func testFGetObjectContextV2() { // initialize logging params startTime := time.Now() testName := getFuncName() @@ -11152,7 +11105,7 @@ func testListObjects() { } } - testList(c.ListObjects, bucketName, minio.ListObjectsOptions{Recursive: true, ForceV1: true}) + testList(c.ListObjects, bucketName, minio.ListObjectsOptions{Recursive: true, UseV1: true}) testList(c.ListObjects, bucketName, minio.ListObjectsOptions{Recursive: true}) // Delete all objects and buckets @@ -11165,11 +11118,11 @@ func testListObjects() { } // Test deleting multiple objects with object retention set in Governance mode -func testRemoveObjectsWithOptions() { +func testRemoveObjects() { // initialize logging params startTime := time.Now() testName := getFuncName() - function := "RemoveObjectsWithOptions(bucketName, objectsCh, opts)" + function := "RemoveObjects(bucketName, objectsCh, opts)" args := map[string]interface{}{ "bucketName": "", "objectPrefix": "", @@ -11236,7 +11189,7 @@ func testRemoveObjectsWithOptions() { go func() { defer close(objectsCh) // List all objects from a bucket-name with a matching prefix. - for object := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{ForceV1: true, Recursive: true}) { + for object := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{UseV1: true, Recursive: true}) { if object.Err != nil { log.Fatalln(object.Err) } @@ -11259,7 +11212,7 @@ func testRemoveObjectsWithOptions() { go func() { defer close(objectsCh1) // List all objects from a bucket-name with a matching prefix. - for object := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{ForceV1: true, Recursive: true}) { + for object := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{UseV1: true, Recursive: true}) { if object.Err != nil { log.Fatalln(object.Err) } @@ -11324,10 +11277,10 @@ func main() { testPutObject0ByteV2() testPutObjectNoLengthV2() testPutObjectsUnknownV2() - testGetObjectWithContextV2() - testFPutObjectWithContextV2() - testFGetObjectWithContextV2() - testPutObjectWithContextV2() + testGetObjectContextV2() + testFPutObjectContextV2() + testFGetObjectContextV2() + testPutObjectContextV2() testMakeBucketError() testMakeBucketRegions() testPutObjectWithMetadata() @@ -11350,17 +11303,17 @@ func main() { testFunctional() testGetObjectModified() testPutObjectUploadSeekedObject() - testGetObjectWithContext() - testFPutObjectWithContext() - testFGetObjectWithContext() - testGetObjectACLWithContext() - testPutObjectWithContext() + testGetObjectContext() + testFPutObjectContext() + testFGetObjectContext() + testGetObjectACLContext() + testPutObjectContext() testStorageClassMetadataPutObject() testStorageClassInvalidMetadataPutObject() testStorageClassMetadataCopyObject() testPutObjectWithContentLanguage() testListObjects() - testRemoveObjectsWithOptions() + testRemoveObjects() testListObjectVersions() testStatObjectWithVersioning() testGetObjectWithVersioning() diff --git a/go.sum b/go.sum index 213814107..6452387c3 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -12,7 +10,6 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= @@ -25,14 +22,11 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLD github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -45,7 +39,6 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= diff --git a/pkg/s3utils/utils.go b/pkg/s3utils/utils.go index aa96a68fb..65f939253 100644 --- a/pkg/s3utils/utils.go +++ b/pkg/s3utils/utils.go @@ -262,6 +262,9 @@ func TagDecode(ctag string) map[string]string { // addition to the percent encoding performed by urlEncodePath() used // here, it also percent encodes '/' (forward slash) func TagEncode(tags map[string]string) string { + if tags == nil { + return "" + } values := url.Values{} for k, v := range tags { values[k] = []string{v} From 9e38611754bdd37ac239be724211a18866a9488d Mon Sep 17 00:00:00 2001 From: Anis Elleuch Date: Fri, 10 Jul 2020 23:11:45 +0100 Subject: [PATCH 179/215] refactor: Make StatObjectOptions as type alias to GetObjectOptions (#1334) It is ugly to always pass GetObjectOptions{} inside StatObjectOptions{} for any specific HEAD operation (such as a specific VersionID). This commit will make StatObjectOptions as type alias to GetObjectOptions --- api-compose-object.go | 2 +- api-get-object-file.go | 2 +- api-get-object.go | 4 ++-- api-get-options.go | 4 +--- functional_tests.go | 36 ++++++++++++++++++------------------ 5 files changed, 23 insertions(+), 25 deletions(-) diff --git a/api-compose-object.go b/api-compose-object.go index 15eaf3d91..a949da861 100644 --- a/api-compose-object.go +++ b/api-compose-object.go @@ -357,7 +357,7 @@ func (c Client) ComposeObject(ctx context.Context, dst CopyDestOptions, srcs ... var totalSize, totalParts int64 var err error for i, src := range srcs { - opts := StatObjectOptions{GetObjectOptions{ServerSideEncryption: encrypt.SSE(src.Encryption), VersionID: src.VersionID}} + opts := StatObjectOptions{ServerSideEncryption: encrypt.SSE(src.Encryption), VersionID: src.VersionID} srcObjectInfos[i], err = c.statObject(context.Background(), src.Bucket, src.Object, opts) if err != nil { return UploadInfo{}, err diff --git a/api-get-object-file.go b/api-get-object-file.go index 25f8d15c5..bccff4578 100644 --- a/api-get-object-file.go +++ b/api-get-object-file.go @@ -63,7 +63,7 @@ func (c Client) FGetObject(ctx context.Context, bucketName, objectName, filePath } // Gather md5sum. - objectStat, err := c.StatObject(ctx, bucketName, objectName, StatObjectOptions{opts}) + objectStat, err := c.StatObject(ctx, bucketName, objectName, StatObjectOptions(opts)) if err != nil { return err } diff --git a/api-get-object.go b/api-get-object.go index 746122f01..2404e7ce7 100644 --- a/api-get-object.go +++ b/api-get-object.go @@ -121,7 +121,7 @@ func (c Client) GetObject(ctx context.Context, bucketName, objectName string, op // Remove range header if already set, for stat Operations to get original file size. delete(opts.headers, "Range") - objectInfo, err = c.statObject(ctx, bucketName, objectName, StatObjectOptions{opts}) + objectInfo, err = c.statObject(ctx, bucketName, objectName, StatObjectOptions(opts)) if err != nil { resCh <- getResponse{ Error: err, @@ -144,7 +144,7 @@ func (c Client) GetObject(ctx context.Context, bucketName, objectName string, op if etag != "" && !snowball { opts.SetMatchETag(etag) } - objectInfo, err := c.statObject(ctx, bucketName, objectName, StatObjectOptions{opts}) + objectInfo, err := c.statObject(ctx, bucketName, objectName, StatObjectOptions(opts)) if err != nil { resCh <- getResponse{ Error: err, diff --git a/api-get-options.go b/api-get-options.go index 6271f2e37..5f3ed3656 100644 --- a/api-get-options.go +++ b/api-get-options.go @@ -35,9 +35,7 @@ type GetObjectOptions struct { // StatObjectOptions are used to specify additional headers or options // during GET info/stat requests. -type StatObjectOptions struct { - GetObjectOptions -} +type StatObjectOptions = GetObjectOptions // Header returns the http.Header representation of the GET options. func (o GetObjectOptions) Header() http.Header { diff --git a/functional_tests.go b/functional_tests.go index d507e584f..4e487f6af 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -831,7 +831,7 @@ func testStatObjectWithVersioning() { } for i := 0; i < len(results); i++ { - opts := minio.StatObjectOptions{minio.GetObjectOptions{VersionID: results[i].VersionID}} + opts := minio.StatObjectOptions{VersionID: results[i].VersionID} statInfo, err := c.StatObject(context.Background(), bucketName, objectName, opts) if err != nil { logError(testName, function, args, startTime, "", "error during HEAD object", err) @@ -7653,7 +7653,7 @@ func testSSECMultipartEncryptedToSSECCopyObjectPart() { } // Stat the object and check its length matches - objInfo, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: srcencryption}}) + objInfo, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcencryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -7705,7 +7705,7 @@ func testSSECMultipartEncryptedToSSECCopyObjectPart() { } // Stat the object and check its length matches - objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) + objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -7809,7 +7809,7 @@ func testSSECEncryptedToSSECCopyObjectPart() { logError(testName, function, args, startTime, "", "PutObject call failed", err) } - st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: srcencryption}}) + st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcencryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -7865,7 +7865,7 @@ func testSSECEncryptedToSSECCopyObjectPart() { } // Stat the object and check its length matches - objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) + objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -7969,7 +7969,7 @@ func testSSECEncryptedToUnencryptedCopyPart() { logError(testName, function, args, startTime, "", "PutObject call failed", err) } - st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: srcencryption}}) + st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcencryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -8024,7 +8024,7 @@ func testSSECEncryptedToUnencryptedCopyPart() { } // Stat the object and check its length matches - objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) + objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -8129,7 +8129,7 @@ func testSSECEncryptedToSSES3CopyObjectPart() { logError(testName, function, args, startTime, "", "PutObject call failed", err) } - st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: srcencryption}}) + st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcencryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -8186,7 +8186,7 @@ func testSSECEncryptedToSSES3CopyObjectPart() { } // Stat the object and check its length matches - objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) + objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -8343,7 +8343,7 @@ func testUnencryptedToSSECCopyObjectPart() { } // Stat the object and check its length matches - objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) + objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -8496,7 +8496,7 @@ func testUnencryptedToUnencryptedCopyPart() { } // Stat the object and check its length matches - objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) + objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -8651,7 +8651,7 @@ func testUnencryptedToSSES3CopyObjectPart() { } // Stat the object and check its length matches - objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) + objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -8754,7 +8754,7 @@ func testSSES3EncryptedToSSECCopyObjectPart() { logError(testName, function, args, startTime, "", "PutObject call failed", err) } - st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: srcEncryption}}) + st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcEncryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -8809,7 +8809,7 @@ func testSSES3EncryptedToSSECCopyObjectPart() { } // Stat the object and check its length matches - objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) + objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -8910,7 +8910,7 @@ func testSSES3EncryptedToUnencryptedCopyPart() { if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } - st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: srcEncryption}}) + st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcEncryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -8963,7 +8963,7 @@ func testSSES3EncryptedToUnencryptedCopyPart() { } // Stat the object and check its length matches - objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) + objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -9065,7 +9065,7 @@ func testSSES3EncryptedToSSES3CopyObjectPart() { if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) } - st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: srcEncryption}}) + st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcEncryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } @@ -9120,7 +9120,7 @@ func testSSES3EncryptedToSSES3CopyObjectPart() { } // Stat the object and check its length matches - objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) + objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) } From b486ba1c04d8ca45b5b67abb321fd35a540e68cf Mon Sep 17 00:00:00 2001 From: gregoryfranklin <45693481+gregoryfranklin@users.noreply.github.com> Date: Fri, 10 Jul 2020 23:12:20 +0100 Subject: [PATCH 180/215] Cache STS Web Identity Credentials (#1322) When using the STSWebIdentity credentials provider from the IAM credentials provider, we need to set the expiry time of the credentials. We, therefore, expose the expiry time in the STSWebIdentity credentials provider so that it can be used by the IAM credentials provider. Calls to IsExpired() on the IAM credentials provider will then work as they would have if it had been called on the underlying STSWebIdentity provider. Therefore caching the credentials as intended. Fixes minio/minio-go#1321 --- pkg/credentials/iam_aws.go | 6 +++++- pkg/credentials/sts_web_identity.go | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/credentials/iam_aws.go b/pkg/credentials/iam_aws.go index a7eb8cb9a..25135e2c9 100644 --- a/pkg/credentials/iam_aws.go +++ b/pkg/credentials/iam_aws.go @@ -104,7 +104,11 @@ func (m *IAM) Retrieve() (Value, error) { }, } - return creds.Retrieve() + stsWebIdentityCreds, err := creds.Retrieve() + if err == nil { + m.SetExpiration(creds.Expiration(), DefaultExpiryWindow) + } + return stsWebIdentityCreds, err case len(os.Getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI")) > 0: if len(endpoint) == 0 { diff --git a/pkg/credentials/sts_web_identity.go b/pkg/credentials/sts_web_identity.go index 216f48ac3..a197161a3 100644 --- a/pkg/credentials/sts_web_identity.go +++ b/pkg/credentials/sts_web_identity.go @@ -174,3 +174,8 @@ func (m *STSWebIdentity) Retrieve() (Value, error) { SignerType: SignatureV4, }, nil } + +// Expiration returns the expiration time of the credentials +func (m *STSWebIdentity) Expiration() time.Time { + return m.expiration +} From a662fe813f9eddbacaf8ec4bfbd18d82976512bc Mon Sep 17 00:00:00 2001 From: poornas Date: Mon, 13 Jul 2020 14:21:37 -0700 Subject: [PATCH 181/215] Add get/remove replication config APIs (#1314) --- README.md | 5 + api-get-bucket-replication.go | 81 ++++++++++ api-put-bucket.go | 90 +++++++++++ api-put-object-streaming.go | 4 + api-put-object.go | 28 +++- constants.go | 2 + docs/API.md | 136 ++++++++++++++++- examples/s3/getbucketreplication.go | 62 ++++++++ examples/s3/removebucketreplication.go | 51 +++++++ examples/s3/setbucketreplication.go | 59 ++++++++ go.mod | 1 + go.sum | 199 +++++++++++++++++++++++++ pkg/replication/replication.go | 91 +++++++++++ pkg/signer/request-signature-v2.go | 1 + 14 files changed, 802 insertions(+), 8 deletions(-) create mode 100644 api-get-bucket-replication.go create mode 100644 examples/s3/getbucketreplication.go create mode 100644 examples/s3/removebucketreplication.go create mode 100644 examples/s3/setbucketreplication.go create mode 100644 pkg/replication/replication.go diff --git a/README.md b/README.md index d0b82c1cc..c740f6ed1 100644 --- a/README.md +++ b/README.md @@ -191,6 +191,11 @@ The full API Reference is available here. * [getbucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketencryption.go) * [deletebucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/deletebucketencryption.go) +### Full Examples : Bucket replication Operations +* [setbucketreplication.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketreplication.go) +* [getbucketreplication.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketreplication.go) +* [removebucketreplication.go](https://github.com/minio/minio-go/blob/master/examples/s3/removebucketreplication.go) + ### Full Examples : Bucket notification Operations * [setbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketnotification.go) * [getbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketnotification.go) diff --git a/api-get-bucket-replication.go b/api-get-bucket-replication.go new file mode 100644 index 000000000..37621f783 --- /dev/null +++ b/api-get-bucket-replication.go @@ -0,0 +1,81 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "encoding/xml" + "io/ioutil" + "net/http" + "net/url" + + "github.com/minio/minio-go/v7/pkg/replication" + "github.com/minio/minio-go/v7/pkg/s3utils" +) + +// GetBucketReplication fetches bucket replication configuration.If config is not +// found, returns empty config with nil error. +func (c Client) GetBucketReplication(ctx context.Context, bucketName string, opts ReplicationReqOptions) (cfg replication.Config, err error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return cfg, err + } + bucketReplicationCfg, err := c.getBucketReplication(ctx, bucketName) + if err != nil { + errResponse := ToErrorResponse(err) + if errResponse.Code == "ReplicationConfigurationNotFoundError" { + return cfg, nil + } + return cfg, err + } + return bucketReplicationCfg, nil +} + +// Request server for current bucket replication config. +func (c Client) getBucketReplication(ctx context.Context, bucketName string) (cfg replication.Config, err error) { + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("replication", "") + + // Execute GET on bucket to get replication config. + resp, err := c.executeMethod(ctx, "GET", requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + }) + + defer closeResponse(resp) + if err != nil { + return cfg, err + } + + if resp.StatusCode != http.StatusOK { + return cfg, httpRespToErrorResponse(resp, bucketName, "") + } + + bucketReplicationBuf, err := ioutil.ReadAll(resp.Body) + if err != nil { + return cfg, err + } + replicationCfg := replication.Config{} + if err := xml.Unmarshal(bucketReplicationBuf, &replicationCfg); err != nil { + return cfg, err + } + + return replicationCfg, err +} diff --git a/api-put-bucket.go b/api-put-bucket.go index 13178e7c6..295a9a45d 100644 --- a/api-put-bucket.go +++ b/api-put-bucket.go @@ -26,6 +26,7 @@ import ( "net/url" "strings" + "github.com/minio/minio-go/v7/pkg/replication" "github.com/minio/minio-go/v7/pkg/s3utils" ) @@ -474,3 +475,92 @@ func (c Client) EnableVersioning(ctx context.Context, bucketName string) error { func (c Client) DisableVersioning(ctx context.Context, bucketName string) error { return c.setVersioning(ctx, bucketName, versionDisableConfig, versionDisableConfigLen, versionDisableConfigMD5Sum, versionDisableConfigSHA256) } + +// ReplicationReqOptions represents options specified by user for Replication call. Options may be added in the future. +type ReplicationReqOptions struct{} + +// RemoveBucketReplication removes a replication config on an existing bucket. +func (c Client) RemoveBucketReplication(ctx context.Context, bucketName string, opts ReplicationReqOptions) error { + return c.removeBucketReplication(ctx, bucketName) +} + +// SetBucketReplication sets a replication config on an existing bucket. +func (c Client) SetBucketReplication(ctx context.Context, bucketName string, cfg replication.Config, opts ReplicationReqOptions) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + // If replication is empty then delete it. + if cfg.Empty() { + return c.removeBucketReplication(ctx, bucketName) + } + // Save the updated replication. + return c.putBucketReplication(ctx, bucketName, cfg) +} + +// Saves a new bucket replication. +func (c Client) putBucketReplication(ctx context.Context, bucketName string, cfg replication.Config) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("replication", "") + replication, err := xml.Marshal(cfg) + if err != nil { + return err + } + // Content-length is mandatory for put replication request + replicationReader := bytes.NewReader(replication) + b, err := ioutil.ReadAll(replicationReader) + if err != nil { + return err + } + reqMetadata := requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: replicationReader, + contentLength: int64(len(b)), + contentMD5Base64: sumMD5Base64(b), + } + // Execute PUT to upload a new bucket replication config. + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, bucketName, "") + } + + return nil +} + +// Remove replication from a bucket. +func (c Client) removeBucketReplication(ctx context.Context, bucketName string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("replication", "") + + // Execute DELETE on objectName. + resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return err + } + return nil +} diff --git a/api-put-object-streaming.go b/api-put-object-streaming.go index 74e43308a..98c289d7b 100644 --- a/api-put-object-streaming.go +++ b/api-put-object-streaming.go @@ -28,6 +28,7 @@ import ( "sort" "strings" + "github.com/google/uuid" "github.com/minio/minio-go/v7/pkg/s3utils" ) @@ -434,6 +435,9 @@ func (c Client) putObjectDo(ctx context.Context, bucketName, objectName string, contentSHA256Hex: sha256Hex, } if opts.ReplicationVersionID != "" { + if _, err := uuid.Parse(opts.ReplicationVersionID); err != nil { + return UploadInfo{}, errInvalidArgument(err.Error()) + } urlValues := make(url.Values) urlValues.Set("versionId", opts.ReplicationVersionID) reqMetadata.queryValues = urlValues diff --git a/api-put-object.go b/api-put-object.go index dbc830bf9..663224feb 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -33,6 +33,23 @@ import ( "golang.org/x/net/http/httpguts" ) +// ReplicationStatus represents replication status of object +type ReplicationStatus string + +const ( + // ReplicationStatusPending indicates replication is pending + ReplicationStatusPending ReplicationStatus = "PENDING" + // ReplicationStatusComplete indicates replication completed ok + ReplicationStatusComplete ReplicationStatus = "COMPLETE" + // ReplicationStatusFailed indicates replication failed + ReplicationStatusFailed ReplicationStatus = "FAILED" +) + +// Empty returns true if no replication status set. +func (r ReplicationStatus) Empty() bool { + return r == "" +} + // PutObjectOptions represents options specified by user for PutObject call type PutObjectOptions struct { UserMetadata map[string]string @@ -54,7 +71,8 @@ type PutObjectOptions struct { SendContentMd5 bool DisableMultipart bool ReplicationVersionID string - ReplicationStatus string + ReplicationStatus ReplicationStatus + ReplicationMTime time.Time } // getNumThreads - gets the number of threads to be used in the multipart @@ -116,10 +134,12 @@ func (opts PutObjectOptions) Header() (header http.Header) { header.Set(amzWebsiteRedirectLocation, opts.WebsiteRedirectLocation) } - if opts.ReplicationStatus != "" { - header.Set(amzBucketReplicationStatus, opts.ReplicationStatus) + if opts.ReplicationStatus.Empty() { + header.Set(amzBucketReplicationStatus, string(opts.ReplicationStatus)) + } + if opts.ReplicationMTime.IsZero() { + header.Set(minIOBucketReplicationSourceMTime, opts.ReplicationMTime.Format(time.RFC3339)) } - if len(opts.UserTags) != 0 { header.Set(amzTaggingHeader, s3utils.TagEncode(opts.UserTags)) } diff --git a/constants.go b/constants.go index af4d44c8f..c87fb3bd0 100644 --- a/constants.go +++ b/constants.go @@ -76,4 +76,6 @@ const ( // Replication status amzBucketReplicationStatus = "X-Amz-Replication-Status" + // Minio specific Replication extension + minIOBucketReplicationSourceMTime = "X-Minio-Source-Mtime" ) diff --git a/docs/API.md b/docs/API.md index b94217f37..3920088e3 100644 --- a/docs/API.md +++ b/docs/API.md @@ -63,9 +63,9 @@ func main() { | [`SetBucketTagging`](#SetBucketTagging) | [`FGetObject`](#FGetObject) | | | [`SetObjectLockConfig`](#SetObjectLockConfig) | | | [`GetBucketTagging`](#GetBucketTagging) | [`ComposeObject`](#ComposeObject) | | | [`GetObjectLockConfig`](#GetObjectLockConfig) | | | [`DeleteBucketTagging`](#DeleteBucketTagging) | | | | [`EnableVersioning`](#EnableVersioning) | | -| | | | | [`DisableVersioning`](#DisableVersioning) | | -| | [`PutObjectRetention`](#PutObjectRetention) | | | [`GetBucketEncryption`](#GetBucketEncryption) | | -| | [`GetObjectRetention`](#GetObjectRetention) | | | [`DeleteBucketEncryption`](#DeleteBucketEncryption) | | +| [`SetBucketReplication`](#SetBucketReplication) | | | | [`DisableVersioning`](#DisableVersioning) | | +| [`GetBucketReplication`](#GetBucketReplication) | [`PutObjectRetention`](#PutObjectRetention) | | | [`GetBucketEncryption`](#GetBucketEncryption) | | +| [`RemoveBucketReplication`](#RemoveBucketReplication) | [`GetObjectRetention`](#GetObjectRetention) | | | [`DeleteBucketEncryption`](#DeleteBucketEncryption) | | | | [`PutObjectLegalHold`](#PutObjectLegalHold) | | | | | | | [`GetObjectLegalHold`](#GetObjectLegalHold) | | | | | | | [`SelectObjectContent`](#SelectObjectContent) | | | | | @@ -537,7 +537,8 @@ __minio.PutObjectOptions__ | `opts.SendContentMd5` | _bool_ | Specify if you'd like to send `content-md5` header with PutObject operation. Note that setting this flag will cause higher memory usage because of in-memory `md5sum` calculation. | | `opts.PartSize` | _uint64_ | Specify a custom part size used for uploading the object | | `opts.ReplicationVersionID` | _string_ | Specify VersionID of object to replicate.This option is intended for internal use by MinIO server to extend the replication API implementation by AWS. This option should not be set unless the application is aware of intended use. | -| `opts.ReplicationStatus` | _string_ | Specify replication status of object. This option is intended for internal use by MinIO server to extend the replication API implementation by AWS. This option should not be set unless the application is aware of intended use. | +| `opts.ReplicationStatus` | _minio.ReplicationStatus_ | Specify replication status of object. This option is intended for internal use by MinIO server to extend the replication API implementation by AWS. This option should not be set unless the application is aware of intended use. | +| `opts.ReplicationMTime` | _time.Time_ | Preserve source modTime on the replicated object. This option is intended for internal use only by MinIO server to comply with AWS bucket replication implementation. This option should not be set unless the application is aware of intended use. | __minio.UploadInfo__ @@ -1838,6 +1839,133 @@ if err != nil { fmt.Printf("%+v\n", versioningConfig) ``` + + +### SetBucketReplication(ctx context.Context, bucketname, cfg replication.Config, opts ReplicationReqOptions) error +Set replication configuration on a bucket. To use this API with MinIO server, ReplicationArn should be set in the replication config. Replication ARN can be obtained by first defining the replication target on MinIO +using `mc admin bucket replication set` to associate the source and destination buckets for replication with the replication endpoint. Next, issue a `mc admin bucket remote` to fetch the replication ARN associated with this replication endpoint. + +__Parameters__ + +|Param |Type |Description | +|:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| +|`bucketName` | _string_ |Name of the bucket| +|`cfg` | _replication.Config_ |Replication configuration to be set | +|`opts` | _ReplicationReqOptions_ |Options for replication - This is currently empty.| + +__Return Values__ + +|Param |Type |Description | +|:---|:---| :---| +|`err` | _error_ |Standard Error | + +__Example__ + +```go +replicationStr := ` + + + + Disabled + + + string + string + + + + string + + string + string + + ... + + string + + string + string + + + string + string + integer + string + +` +replicationConfig := replication.Config{} +if err := xml.Unmarshal([]byte(replicationStr), &replicationConfig); err != nil { + log.Fatalln(err) +} +// this is optional for replication with MinIO server. +cfg.ReplicationArn := "arn:minio:s3::598361bf-3cec-49a7-b529-ce870a34d759:*" +err = minioClient.SetBucketReplication(context.Background(), "my-bucketname", replicationConfig, ReplicationReqOptions{}) +if err != nil { + fmt.Println(err) + return +} +``` + + + +### GetBucketReplication(ctx context.Context, bucketName string, opts ReplicationReqOptions) (replication.Config, error) +Get current replication config on a bucket. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| +|`bucketName` | _string_ |Name of the bucket | +|`opts` | _ReplicationReqOptions_ | options for replication request + +__Return Values__ + + +|Param |Type |Description | +|:---|:---| :---| +|`replication` | _replication.Config_ |Replication config returned from the server | +|`err` | _error_ |Standard Error | + +__Example__ + +```go +replication, err := minioClient.GetBucketReplication(context.Background(), "my-bucketname", ReplicationReqOptions{}) +if err != nil { + log.Fatalln(err) +} +``` + + + +### RemoveBucketReplication(ctx context.Context, bucketname string, opts ReplicationReqOptions) error +Removes replication configuration on a bucket. +__Parameters__ + +|Param |Type |Description | +|:---|:---| :---| +|`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| +|`bucketName` | _string_ |Name of the bucket| +|`opts` | _ReplicationReqOptions_ | options for replication request + +__Return Values__ + +|Param |Type |Description | +|:---|:---| :---| +|`err` | _error_ |Standard Error | + +__Example__ + +```go +err = minioClient.RemoveBucketReplication(context.Background(), "my-bucketname", ReplicationReqOptions{}) +if err != nil { + fmt.Println(err) + return +} +``` + ## 7. Client custom settings diff --git a/examples/s3/getbucketreplication.go b/examples/s3/getbucketreplication.go new file mode 100644 index 000000000..d7f9a95c0 --- /dev/null +++ b/examples/s3/getbucketreplication.go @@ -0,0 +1,62 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "context" + "encoding/json" + "log" + "os" + + "github.com/minio/minio-go/v7" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + // s3Client.TraceOn(os.Stderr) + // Get bucket replication configuration from S3 + replicationCfg, err := s3Client.GetBucketReplication(context.Background(), "my-bucketname") + if err != nil { + log.Fatalln(err) + } + // Create replication config file + localReplicationCfgFile, err := os.Create("replication.xml") + if err != nil { + log.Fatalln(err) + } + defer localReplicationCfgFile.Close() + + replBytes, err := json.Marshal(replicationCfg) + if err != nil { + log.Fatalln(err) + } + localReplicationCfgFile.Write(replBytes) +} diff --git a/examples/s3/removebucketreplication.go b/examples/s3/removebucketreplication.go new file mode 100644 index 000000000..c8769f110 --- /dev/null +++ b/examples/s3/removebucketreplication.go @@ -0,0 +1,51 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "context" + "log" + + "github.com/minio/minio-go/v7" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // s3Client.TraceOn(os.Stderr) + + // Remove replication configuration on a bucket + err = s3Client.RemoveBucketReplication(context.Background(), "my-bucketname") + + if err != nil { + log.Fatalln(err) + } +} diff --git a/examples/s3/setbucketreplication.go b/examples/s3/setbucketreplication.go new file mode 100644 index 000000000..428e7dab0 --- /dev/null +++ b/examples/s3/setbucketreplication.go @@ -0,0 +1,59 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "context" + "encoding/xml" + "log" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/replication" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + // s3Client.TraceOn(os.Stderr) + replicationStr := `stringEnabled1Disabledarn:aws:s3:::destPrefixTag-Key1Tag-Value1Tag-Key2Tag-Value2` + var replCfg replication.Config + err = xml.Unmarshal([]byte(replicationStr), &replCfg) + if err != nil { + log.Fatalln(err) + } + + // This replication ARN should have been generated for replication endpoint using `mc admin bucket remote` command + replCfg.replicationArn = "arn:minio:s3::dadddae7-f1d7-440f-b5d6-651aa9a8c8a7:*" + // Set replication config on a bucket + err = s3Client.SetBucketReplication(context.Background(), "my-bucketname", replCfg, ReplicationReqOptions{}) + if err != nil { + log.Fatalln(err) + } +} diff --git a/go.mod b/go.mod index bdfb04133..8b6e120a2 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/minio/minio-go/v7 go 1.12 require ( + github.com/google/uuid v1.1.1 github.com/json-iterator/go v1.1.10 github.com/minio/md5-simd v1.1.0 github.com/minio/sha256-simd v0.1.1 diff --git a/go.sum b/go.sum index 6452387c3..2406460b8 100644 --- a/go.sum +++ b/go.sum @@ -1,48 +1,247 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.39.0/go.mod h1:rVLT6fkc8chs9sfPtFc1SBH6em7n+ZoXaG+87tDISts= +contrib.go.opencensus.io/exporter/ocagent v0.5.0/go.mod h1:ImxhfLRpxoYiSq891pBrLVhN+qmP8BTVvdH2YLs7Gl0= +git.apache.org/thrift.git v0.13.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= +github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0= +github.com/Azure/go-autorest v11.7.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/Shopify/sarama v1.24.1/go.mod h1:fGP8eQ6PugKEI0iUETYYtnP6d1pH/bdDMTel1X5ajsU= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/alecthomas/participle v0.2.1/go.mod h1:SW6HZGeZgSIpcUWX3fXpfZhuaWHnmoD5KCVaqSaNTkk= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aws/aws-sdk-go v1.20.21/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2/go.mod h1:RDu/qcrnpEdJC/p8tx34+YBFqqX71lB7dOX9QE+ZC4M= +github.com/beevik/ntp v0.2.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cheggaaa/pb v1.0.28/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= +github.com/coredns/coredns v1.4.0/go.mod h1:zASH/MVDgR6XZTbxvOnsZfffS+31vg6Ackf/wo1+AM0= +github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.12+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.4/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.10.1/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/cpuid v1.2.2/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/simdjson-go v0.1.5-0.20200303142138-b17fe061ea37/go.mod h1:oKURrZZEBtqObgJrSjN1Ln2n9MJj2icuBTkeJzZnvSI= +github.com/minio/sio v0.2.0 h1:NCRCFLx0r5pRbXf65LVNjxbCGZgNQvNFQkgX3XF4BoA= +github.com/minio/sio v0.2.0/go.mod h1:nKM5GIWSrqbOZp0uhyj6M1iA0X6xQzSGtYSaTKSCut0= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mmcloughlin/avo v0.0.0-20200303042253-6df701fe672f/go.mod h1:L0u9qfRMLNBO97u6pPukRp6ncoQz0Q25W69fvtht3vA= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/montanaflynn/stats v0.5.0 h1:2EkzeTSqBB4V4bJwWrt5gIIrZmpJBcoIRGS2kWLgzmk= +github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nats-io/gnatsd v1.4.1/go.mod h1:nqco77VO78hLCJpIcVfygDP2rPGfsEHkGTUk94uh5DQ= +github.com/nats-io/go-nats v1.7.2/go.mod h1:+t7RHT5ApZebkrQdnn6AhQJmhJJiKAvJUio1PiiCtj0= +github.com/nats-io/go-nats-streaming v0.4.4/go.mod h1:gfq4R3c9sKAINOpelo0gn/b9QDMBZnmrttcsNF+lqyo= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/nats-server v1.4.1/go.mod h1:c8f/fHd2B6Hgms3LtCaI7y6pC4WD1f4SUxcCud5vhBc= +github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= +github.com/nats-io/nats-streaming-server v0.14.2/go.mod h1:RyqtDJZvMZO66YmyjIYdIvS69zu/wDAkyNWa8PIUa5c= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/nats-io/stan.go v0.4.5/go.mod h1:Ji7mK6gRZJSH1nc3ZJH6vi7zn/QnZhpR9Arm4iuzsUQ= +github.com/ncw/directio v1.0.5 h1:JSUBhdjEvVaJvOoyPAbcW0fnd0tvRXD76wEfZ1KcQz4= +github.com/ncw/directio v1.0.5/go.mod h1:rX/pKEYkOXBGOggmcyJeJGloCkleSvphPx2eV3t6ROk= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nsqio/go-nsq v1.0.7/go.mod h1:XP5zaUs3pqf+Q71EqUJs3HYfBIqfK6G83WQMdNN+Ito= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= +github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.2.6+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.4.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/tidwall/gjson v1.3.5/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= +github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/sjson v1.0.4/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= +github.com/tinylib/msgp v1.1.1/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/ugorji/go v1.1.5-pre/go.mod h1:FwP/aQVg39TXzItUBMwnWp9T9gPQnXw4Poh4/oBQZ/0= +github.com/ugorji/go/codec v1.1.5-pre/go.mod h1:tULtS6Gy1AE1yCENaw4Vb//HLH5njI2tfCQDUqRd8fI= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/willf/bloom v2.0.3+incompatible/go.mod h1:MmAltL9pDMNTrvUkxdg0k0q5I0suxmuwp3KbyrZLOZ8= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= +golang.org/x/arch v0.0.0-20190909030613-46d78d1859ac/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4= +golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 h1:vEg9joUBmeBcK9iSJftGNf3coIG4HqZElCPehJsfAYM= +golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200610111108-226ff32320da h1:bGb80FudwxpeucJUjPYJXuJ8Hk91vNtfvrymzwiei38= +golang.org/x/sys v0.0.0-20200610111108-226ff32320da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/pkg/replication/replication.go b/pkg/replication/replication.go new file mode 100644 index 000000000..f4818e992 --- /dev/null +++ b/pkg/replication/replication.go @@ -0,0 +1,91 @@ +/* + * MinIO Client (C) 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package replication + +import "encoding/xml" + +// Config - replication configuration specified in +// https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html +type Config struct { + XMLName xml.Name `xml:"ReplicationConfiguration" json:"-"` + Rules []Rule `xml:"Rule" json:"Rules"` + // ReplicationArn is a MinIO only extension and optional for AWS + ReplicationArn string `xml:"ReplicationArn,omitempty" json:"ReplicationArn,omitempty"` +} + +// Empty returns true if config is not set +func (c *Config) Empty() bool { + return len(c.Rules) == 0 +} + +// Rule - a rule for replication configuration. +type Rule struct { + XMLName xml.Name `xml:"Rule" json:"-"` + ID string `xml:"ID,omitempty"` + Status Status `xml:"Status"` + Priority int `xml:"Priority"` + DeleteMarkerReplication DeleteMarkerReplication `xml:"DeleteMarkerReplication"` + Destination Destination `xml:"Destination"` + Filter Filter `xml:"Filter" json:"Filter"` +} + +// Filter - a filter for a replication configuration Rule. +type Filter struct { + XMLName xml.Name `xml:"Filter" json:"-"` + Prefix string `json:"Prefix,omitempty"` + And And `xml:"And,omitempty" json:"And,omitempty"` +} + +// Tag - a tag for a replication configuration Rule filter. +type Tag struct { + XMLName xml.Name `json:"-"` + Key string `xml:"Key,omitempty" json:"Key,omitempty"` + Value string `xml:"Value,omitempty" json:"Value,omitempty"` +} + +// Destination - destination in ReplicationConfiguration. +type Destination struct { + XMLName xml.Name `xml:"Destination" json:"-"` + Bucket string `xml:"Bucket" json:"Bucket"` + StorageClass string `xml:"StorageClass,omitempty" json:"StorageClass,omitempty"` +} + +// And - a tag to combine a prefix and multiple tags for replication configuration rule. +type And struct { + XMLName xml.Name `xml:"And,omitempty" json:"-"` + Prefix string `xml:"Prefix,omitempty" json:"Prefix,omitempty"` + Tags []Tag `xml:"Tag,omitempty" json:"Tags,omitempty"` +} + +// Status represents Enabled/Disabled status +type Status string + +// Supported status types +const ( + Enabled Status = "Enabled" + Disabled Status = "Disabled" +) + +// DeleteMarkerReplication - whether delete markers are replicated - https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html +type DeleteMarkerReplication struct { + Status Status `xml:"Status" json:"Status"` // should be set to "Disabled" by default +} + +// IsEmpty returns true if DeleteMarkerReplication is not set +func (d DeleteMarkerReplication) IsEmpty() bool { + return len(d.Status) == 0 +} diff --git a/pkg/signer/request-signature-v2.go b/pkg/signer/request-signature-v2.go index d908f2779..71821a26a 100644 --- a/pkg/signer/request-signature-v2.go +++ b/pkg/signer/request-signature-v2.go @@ -262,6 +262,7 @@ var resourceList = []string{ "notification", "partNumber", "policy", + "replication", "requestPayment", "response-cache-control", "response-content-disposition", From 51e20555ab88a1e7ccd52b186dead5496d351aa8 Mon Sep 17 00:00:00 2001 From: poornas Date: Mon, 13 Jul 2020 15:23:59 -0700 Subject: [PATCH 182/215] Add `Replica` type to replication status (#1335) --- api-put-object.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api-put-object.go b/api-put-object.go index 663224feb..a955fded1 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -43,6 +43,8 @@ const ( ReplicationStatusComplete ReplicationStatus = "COMPLETE" // ReplicationStatusFailed indicates replication failed ReplicationStatusFailed ReplicationStatus = "FAILED" + // ReplicationStatusReplica indicates object is a replica of a source + ReplicationStatusReplica ReplicationStatus = "REPLICA" ) // Empty returns true if no replication status set. From 47e386e2cde851f8137ef6477b72d7334c5035f8 Mon Sep 17 00:00:00 2001 From: ebozduman Date: Tue, 14 Jul 2020 01:55:48 -0700 Subject: [PATCH 183/215] Returns error instead of nil when an error is hit (#1336) --- api-get-object.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-get-object.go b/api-get-object.go index 2404e7ce7..78147a97a 100644 --- a/api-get-object.go +++ b/api-get-object.go @@ -638,7 +638,7 @@ func (c Client) getObject(ctx context.Context, bucketName, objectName string, op objectStat, err := ToObjectInfo(bucketName, objectName, resp.Header) if err != nil { closeResponse(resp) - return nil, objectStat, resp.Header, nil + return nil, ObjectInfo{}, nil, err } // do not close body here, caller will close From 319f6cf993beffa4a4067a693e6d1753e89f4e30 Mon Sep 17 00:00:00 2001 From: poornas Date: Tue, 14 Jul 2020 16:36:10 -0700 Subject: [PATCH 184/215] set replication opts correctly (#1337) --- api-put-object.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-put-object.go b/api-put-object.go index a955fded1..8cbe004d2 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -136,10 +136,10 @@ func (opts PutObjectOptions) Header() (header http.Header) { header.Set(amzWebsiteRedirectLocation, opts.WebsiteRedirectLocation) } - if opts.ReplicationStatus.Empty() { + if !opts.ReplicationStatus.Empty() { header.Set(amzBucketReplicationStatus, string(opts.ReplicationStatus)) } - if opts.ReplicationMTime.IsZero() { + if !opts.ReplicationMTime.IsZero() { header.Set(minIOBucketReplicationSourceMTime, opts.ReplicationMTime.Format(time.RFC3339)) } if len(opts.UserTags) != 0 { From 5f85da59959d7f5e77c4ebd49b8684b7cbaedd64 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 15 Jul 2020 14:57:17 -0700 Subject: [PATCH 185/215] reimplement all Bucket config APIs (#1338) --- api-bucket-encryption.go | 134 ++++++ api-bucket-lifecycle.go | 147 ++++++ ...ification.go => api-bucket-notification.go | 149 +++--- api-get-policy.go => api-bucket-policy.go | 68 ++- api-bucket-replication.go | 149 ++++++ api-bucket-tagging.go | 4 +- ...-versioning.go => api-bucket-versioning.go | 62 ++- api-get-bucket-encryption.go | 67 --- api-get-bucket-replication.go | 81 ---- api-get-lifecycle.go | 77 --- api-put-bucket.go | 443 ------------------ docs/API.md | 151 +++--- ...ncryption.go => removebucketencryption.go} | 2 +- examples/s3/removebuckettagging.go | 2 +- examples/s3/setbucketencryption.go | 12 +- examples/s3/setbucketlifecycle.go | 12 +- examples/s3/setbucketnotification.go | 23 +- examples/s3/setbucketreplication.go | 6 +- functional_tests.go | 34 +- go.mod | 2 + go.sum | 204 +------- pkg/lifecycle/lifecycle.go | 232 +++++++++ pkg/notification/info.go | 78 +++ .../notification/notification.go | 118 ++--- .../notification/notification_test.go | 126 ++--- pkg/replication/replication.go | 4 +- pkg/sse/sse.go | 66 +++ 27 files changed, 1243 insertions(+), 1210 deletions(-) create mode 100644 api-bucket-encryption.go create mode 100644 api-bucket-lifecycle.go rename api-notification.go => api-bucket-notification.go (61%) rename api-get-policy.go => api-bucket-policy.go (53%) create mode 100644 api-bucket-replication.go rename api-get-bucket-versioning.go => api-bucket-versioning.go (56%) delete mode 100644 api-get-bucket-encryption.go delete mode 100644 api-get-bucket-replication.go delete mode 100644 api-get-lifecycle.go rename examples/s3/{deletebucketencryption.go => removebucketencryption.go} (95%) create mode 100644 pkg/lifecycle/lifecycle.go create mode 100644 pkg/notification/info.go rename bucket-notification.go => pkg/notification/notification.go (68%) rename bucket-notification_test.go => pkg/notification/notification_test.go (83%) create mode 100644 pkg/sse/sse.go diff --git a/api-bucket-encryption.go b/api-bucket-encryption.go new file mode 100644 index 000000000..e02ab84af --- /dev/null +++ b/api-bucket-encryption.go @@ -0,0 +1,134 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "context" + "encoding/xml" + "net/http" + "net/url" + + "github.com/minio/minio-go/v7/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/sse" +) + +// SetBucketEncryption sets the default encryption configuration on an existing bucket. +func (c Client) SetBucketEncryption(ctx context.Context, bucketName string, config *sse.Configuration) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + if config == nil { + return errInvalidArgument("configuration cannot be empty") + } + + buf, err := xml.Marshal(config) + if err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("encryption", "") + + // Content-length is mandatory to set a default encryption configuration + reqMetadata := requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: bytes.NewReader(buf), + contentLength: int64(len(buf)), + contentMD5Base64: sumMD5Base64(buf), + } + + // Execute PUT to upload a new bucket default encryption configuration. + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, bucketName, "") + } + return nil +} + +// RemoveBucketEncryption removes the default encryption configuration on a bucket with a context to control cancellations and timeouts. +func (c Client) RemoveBucketEncryption(ctx context.Context, bucketName string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("encryption", "") + + // DELETE default encryption configuration on a bucket. + resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return httpRespToErrorResponse(resp, bucketName, "") + } + return nil +} + +// GetBucketEncryption gets the default encryption configuration +// on an existing bucket with a context to control cancellations and timeouts. +func (c Client) GetBucketEncryption(ctx context.Context, bucketName string) (*sse.Configuration, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return nil, err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("encryption", "") + + // Execute GET on bucket to get the default encryption configuration. + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + }) + + defer closeResponse(resp) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return nil, httpRespToErrorResponse(resp, bucketName, "") + } + + encryptionConfig := &sse.Configuration{} + if err = xmlDecoder(resp.Body, encryptionConfig); err != nil { + return nil, err + } + + return encryptionConfig, nil +} diff --git a/api-bucket-lifecycle.go b/api-bucket-lifecycle.go new file mode 100644 index 000000000..daaceb52f --- /dev/null +++ b/api-bucket-lifecycle.go @@ -0,0 +1,147 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "context" + "encoding/xml" + "io/ioutil" + "net/http" + "net/url" + + "github.com/minio/minio-go/v7/pkg/lifecycle" + "github.com/minio/minio-go/v7/pkg/s3utils" +) + +// SetBucketLifecycle set the lifecycle on an existing bucket. +func (c Client) SetBucketLifecycle(ctx context.Context, bucketName string, config *lifecycle.Configuration) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + // If lifecycle is empty then delete it. + if config == nil { + return c.removeBucketLifecycle(ctx, bucketName) + } + + buf, err := xml.Marshal(config) + if err != nil { + return err + } + + // Save the updated lifecycle. + return c.putBucketLifecycle(ctx, bucketName, buf) +} + +// Saves a new bucket lifecycle. +func (c Client) putBucketLifecycle(ctx context.Context, bucketName string, buf []byte) error { + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("lifecycle", "") + + // Content-length is mandatory for put lifecycle request + reqMetadata := requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: bytes.NewReader(buf), + contentLength: int64(len(buf)), + contentMD5Base64: sumMD5Base64(buf), + } + + // Execute PUT to upload a new bucket lifecycle. + resp, err := c.executeMethod(ctx, "PUT", reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, bucketName, "") + } + } + return nil +} + +// Remove lifecycle from a bucket. +func (c Client) removeBucketLifecycle(ctx context.Context, bucketName string) error { + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("lifecycle", "") + + // Execute DELETE on objectName. + resp, err := c.executeMethod(ctx, "DELETE", requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return err + } + return nil +} + +// GetBucketLifecycle fetch bucket lifecycle configuration +func (c Client) GetBucketLifecycle(ctx context.Context, bucketName string) (*lifecycle.Configuration, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return nil, err + } + + bucketLifecycle, err := c.getBucketLifecycle(ctx, bucketName) + if err != nil { + return nil, err + } + + config := lifecycle.NewConfiguration() + if err = xml.Unmarshal(bucketLifecycle, config); err != nil { + return nil, err + } + return config, nil +} + +// Request server for current bucket lifecycle. +func (c Client) getBucketLifecycle(ctx context.Context, bucketName string) ([]byte, error) { + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("lifecycle", "") + + // Execute GET on bucket to get lifecycle. + resp, err := c.executeMethod(ctx, "GET", requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + }) + + defer closeResponse(resp) + if err != nil { + return nil, err + } + + if resp != nil { + if resp.StatusCode != http.StatusOK { + return nil, httpRespToErrorResponse(resp, bucketName, "") + } + } + + return ioutil.ReadAll(resp.Body) +} diff --git a/api-notification.go b/api-bucket-notification.go similarity index 61% rename from api-notification.go rename to api-bucket-notification.go index 81d6efe88..8f198b116 100644 --- a/api-notification.go +++ b/api-bucket-notification.go @@ -19,30 +19,75 @@ package minio import ( "bufio" + "bytes" "context" + "encoding/xml" "net/http" "net/url" "time" jsoniter "github.com/json-iterator/go" + "github.com/minio/minio-go/v7/pkg/notification" "github.com/minio/minio-go/v7/pkg/s3utils" ) -// GetBucketNotification returns current bucket notification configuration -func (c Client) GetBucketNotification(ctx context.Context, bucketName string) (bucketNotification BucketNotification, err error) { +// SetBucketNotification saves a new bucket notification with a context to control cancellations and timeouts. +func (c Client) SetBucketNotification(ctx context.Context, bucketName string, config notification.Configuration) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return BucketNotification{}, err + return err } - notification, err := c.getBucketNotification(ctx, bucketName) + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("notification", "") + + notifBytes, err := xml.Marshal(&config) if err != nil { - return BucketNotification{}, err + return err } - return notification, nil + + notifBuffer := bytes.NewReader(notifBytes) + reqMetadata := requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: notifBuffer, + contentLength: int64(len(notifBytes)), + contentMD5Base64: sumMD5Base64(notifBytes), + contentSHA256Hex: sum256Hex(notifBytes), + } + + // Execute PUT to upload a new bucket notification. + resp, err := c.executeMethod(ctx, "PUT", reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, bucketName, "") + } + } + return nil +} + +// RemoveAllBucketNotification - Remove bucket notification clears all previously specified config +func (c Client) RemoveAllBucketNotification(ctx context.Context, bucketName string) error { + return c.SetBucketNotification(ctx, bucketName, notification.Configuration{}) +} + +// GetBucketNotification returns current bucket notification configuration +func (c Client) GetBucketNotification(ctx context.Context, bucketName string) (bucketNotification notification.Configuration, err error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return notification.Configuration{}, err + } + return c.getBucketNotification(ctx, bucketName) } // Request server for notification rules. -func (c Client) getBucketNotification(ctx context.Context, bucketName string) (BucketNotification, error) { +func (c Client) getBucketNotification(ctx context.Context, bucketName string) (notification.Configuration, error) { urlValues := make(url.Values) urlValues.Set("notification", "") @@ -55,99 +100,39 @@ func (c Client) getBucketNotification(ctx context.Context, bucketName string) (B defer closeResponse(resp) if err != nil { - return BucketNotification{}, err + return notification.Configuration{}, err } return processBucketNotificationResponse(bucketName, resp) } // processes the GetNotification http response from the server. -func processBucketNotificationResponse(bucketName string, resp *http.Response) (BucketNotification, error) { +func processBucketNotificationResponse(bucketName string, resp *http.Response) (notification.Configuration, error) { if resp.StatusCode != http.StatusOK { errResponse := httpRespToErrorResponse(resp, bucketName, "") - return BucketNotification{}, errResponse + return notification.Configuration{}, errResponse } - var bucketNotification BucketNotification + var bucketNotification notification.Configuration err := xmlDecoder(resp.Body, &bucketNotification) if err != nil { - return BucketNotification{}, err + return notification.Configuration{}, err } return bucketNotification, nil } -// Indentity represents the user id, this is a compliance field. -type identity struct { - PrincipalID string `json:"principalId"` -} - -// Notification event bucket metadata. -type bucketMeta struct { - Name string `json:"name"` - OwnerIdentity identity `json:"ownerIdentity"` - ARN string `json:"arn"` -} - -// Notification event object metadata. -type objectMeta struct { - Key string `json:"key"` - Size int64 `json:"size,omitempty"` - ETag string `json:"eTag,omitempty"` - ContentType string `json:"contentType,omitempty"` - UserMetadata map[string]string `json:"userMetadata,omitempty"` - VersionID string `json:"versionId,omitempty"` - Sequencer string `json:"sequencer"` -} - -// Notification event server specific metadata. -type eventMeta struct { - SchemaVersion string `json:"s3SchemaVersion"` - ConfigurationID string `json:"configurationId"` - Bucket bucketMeta `json:"bucket"` - Object objectMeta `json:"object"` -} - -// sourceInfo represents information on the client that -// triggered the event notification. -type sourceInfo struct { - Host string `json:"host"` - Port string `json:"port"` - UserAgent string `json:"userAgent"` -} - -// NotificationEvent represents an Amazon an S3 bucket notification event. -type NotificationEvent struct { - EventVersion string `json:"eventVersion"` - EventSource string `json:"eventSource"` - AwsRegion string `json:"awsRegion"` - EventTime string `json:"eventTime"` - EventName string `json:"eventName"` - UserIdentity identity `json:"userIdentity"` - RequestParameters map[string]string `json:"requestParameters"` - ResponseElements map[string]string `json:"responseElements"` - S3 eventMeta `json:"s3"` - Source sourceInfo `json:"source"` -} - -// NotificationInfo - represents the collection of notification events, additionally -// also reports errors if any while listening on bucket notifications. -type NotificationInfo struct { - Records []NotificationEvent - Err error -} - // ListenBucketNotification listen for bucket events, this is a MinIO specific API -func (c Client) ListenBucketNotification(ctx context.Context, bucketName, prefix, suffix string, events []string) <-chan NotificationInfo { - notificationInfoCh := make(chan NotificationInfo, 1) +func (c Client) ListenBucketNotification(ctx context.Context, bucketName, prefix, suffix string, events []string) <-chan notification.Info { + notificationInfoCh := make(chan notification.Info, 1) const notificationCapacity = 1024 * 1024 notificationEventBuffer := make([]byte, notificationCapacity) // Only success, start a routine to start reading line by line. - go func(notificationInfoCh chan<- NotificationInfo) { + go func(notificationInfoCh chan<- notification.Info) { defer close(notificationInfoCh) // Validate the bucket name. if err := s3utils.CheckValidBucketName(bucketName); err != nil { select { - case notificationInfoCh <- NotificationInfo{ + case notificationInfoCh <- notification.Info{ Err: err, }: case <-ctx.Done(): @@ -158,7 +143,7 @@ func (c Client) ListenBucketNotification(ctx context.Context, bucketName, prefix // Check ARN partition to verify if listening bucket is supported if s3utils.IsAmazonEndpoint(*c.endpointURL) || s3utils.IsGoogleEndpoint(*c.endpointURL) { select { - case notificationInfoCh <- NotificationInfo{ + case notificationInfoCh <- notification.Info{ Err: errAPINotSupported("Listening for bucket notification is specific only to `minio` server endpoints"), }: case <-ctx.Done(): @@ -189,7 +174,7 @@ func (c Client) ListenBucketNotification(ctx context.Context, bucketName, prefix }) if err != nil { select { - case notificationInfoCh <- NotificationInfo{ + case notificationInfoCh <- notification.Info{ Err: err, }: case <-ctx.Done(): @@ -201,7 +186,7 @@ func (c Client) ListenBucketNotification(ctx context.Context, bucketName, prefix if resp.StatusCode != http.StatusOK { errResponse := httpRespToErrorResponse(resp, bucketName, "") select { - case notificationInfoCh <- NotificationInfo{ + case notificationInfoCh <- notification.Info{ Err: errResponse, }: case <-ctx.Done(): @@ -219,12 +204,12 @@ func (c Client) ListenBucketNotification(ctx context.Context, bucketName, prefix // Unmarshal each line, returns marshaled values. for bio.Scan() { - var notificationInfo NotificationInfo + var notificationInfo notification.Info if err = json.Unmarshal(bio.Bytes(), ¬ificationInfo); err != nil { // Unexpected error during json unmarshal, send // the error to caller for actionable as needed. select { - case notificationInfoCh <- NotificationInfo{ + case notificationInfoCh <- notification.Info{ Err: err, }: case <-ctx.Done(): @@ -244,7 +229,7 @@ func (c Client) ListenBucketNotification(ctx context.Context, bucketName, prefix if err = bio.Err(); err != nil { select { - case notificationInfoCh <- NotificationInfo{ + case notificationInfoCh <- notification.Info{ Err: err, }: case <-ctx.Done(): diff --git a/api-get-policy.go b/api-bucket-policy.go similarity index 53% rename from api-get-policy.go rename to api-bucket-policy.go index 859c63901..8b3a20034 100644 --- a/api-get-policy.go +++ b/api-bucket-policy.go @@ -1,7 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. - * + * Copyright 2020 MinIO, Inc. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -22,10 +21,75 @@ import ( "io/ioutil" "net/http" "net/url" + "strings" "github.com/minio/minio-go/v7/pkg/s3utils" ) +// SetBucketPolicy sets the access permissions on an existing bucket. +func (c Client) SetBucketPolicy(ctx context.Context, bucketName, policy string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + // If policy is empty then delete the bucket policy. + if policy == "" { + return c.removeBucketPolicy(ctx, bucketName) + } + + // Save the updated policies. + return c.putBucketPolicy(ctx, bucketName, policy) +} + +// Saves a new bucket policy. +func (c Client) putBucketPolicy(ctx context.Context, bucketName, policy string) error { + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("policy", "") + + reqMetadata := requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: strings.NewReader(policy), + contentLength: int64(len(policy)), + } + + // Execute PUT to upload a new bucket policy. + resp, err := c.executeMethod(ctx, "PUT", reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp != nil { + if resp.StatusCode != http.StatusNoContent { + return httpRespToErrorResponse(resp, bucketName, "") + } + } + return nil +} + +// Removes all policies on a bucket. +func (c Client) removeBucketPolicy(ctx context.Context, bucketName string) error { + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("policy", "") + + // Execute DELETE on objectName. + resp, err := c.executeMethod(ctx, "DELETE", requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return err + } + return nil +} + // GetBucketPolicy returns the current policy func (c Client) GetBucketPolicy(ctx context.Context, bucketName string) (string, error) { // Input validation. diff --git a/api-bucket-replication.go b/api-bucket-replication.go new file mode 100644 index 000000000..116a17ee1 --- /dev/null +++ b/api-bucket-replication.go @@ -0,0 +1,149 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "context" + "encoding/xml" + "net/http" + "net/url" + + "github.com/minio/minio-go/v7/pkg/replication" + "github.com/minio/minio-go/v7/pkg/s3utils" +) + +// RemoveBucketReplication removes a replication config on an existing bucket. +func (c Client) RemoveBucketReplication(ctx context.Context, bucketName string) error { + return c.removeBucketReplication(ctx, bucketName) +} + +// SetBucketReplication sets a replication config on an existing bucket. +func (c Client) SetBucketReplication(ctx context.Context, bucketName string, cfg replication.Config) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + // If replication is empty then delete it. + if cfg.Empty() { + return c.removeBucketReplication(ctx, bucketName) + } + // Save the updated replication. + return c.putBucketReplication(ctx, bucketName, cfg) +} + +// Saves a new bucket replication. +func (c Client) putBucketReplication(ctx context.Context, bucketName string, cfg replication.Config) error { + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("replication", "") + replication, err := xml.Marshal(cfg) + if err != nil { + return err + } + + reqMetadata := requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: bytes.NewReader(replication), + contentLength: int64(len(replication)), + contentMD5Base64: sumMD5Base64(replication), + } + + // Execute PUT to upload a new bucket replication config. + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, bucketName, "") + } + + return nil +} + +// Remove replication from a bucket. +func (c Client) removeBucketReplication(ctx context.Context, bucketName string) error { + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("replication", "") + + // Execute DELETE on objectName. + resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return err + } + return nil +} + +// GetBucketReplication fetches bucket replication configuration.If config is not +// found, returns empty config with nil error. +func (c Client) GetBucketReplication(ctx context.Context, bucketName string) (cfg replication.Config, err error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return cfg, err + } + bucketReplicationCfg, err := c.getBucketReplication(ctx, bucketName) + if err != nil { + errResponse := ToErrorResponse(err) + if errResponse.Code == "ReplicationConfigurationNotFoundError" { + return cfg, nil + } + return cfg, err + } + return bucketReplicationCfg, nil +} + +// Request server for current bucket replication config. +func (c Client) getBucketReplication(ctx context.Context, bucketName string) (cfg replication.Config, err error) { + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("replication", "") + + // Execute GET on bucket to get replication config. + resp, err := c.executeMethod(ctx, "GET", requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + }) + + defer closeResponse(resp) + if err != nil { + return cfg, err + } + + if resp.StatusCode != http.StatusOK { + return cfg, httpRespToErrorResponse(resp, bucketName, "") + } + + if err = xmlDecoder(resp.Body, &cfg); err != nil { + return cfg, err + } + + return cfg, nil +} diff --git a/api-bucket-tagging.go b/api-bucket-tagging.go index ded968b94..fcb966e63 100644 --- a/api-bucket-tagging.go +++ b/api-bucket-tagging.go @@ -105,9 +105,9 @@ func (c Client) SetBucketTagging(ctx context.Context, bucketName string, tags *t return nil } -// DeleteBucketTagging removes tagging configuration for a +// RemoveBucketTagging removes tagging configuration for a // bucket with a context to control cancellations and timeouts. -func (c Client) DeleteBucketTagging(ctx context.Context, bucketName string) error { +func (c Client) RemoveBucketTagging(ctx context.Context, bucketName string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err diff --git a/api-get-bucket-versioning.go b/api-bucket-versioning.go similarity index 56% rename from api-get-bucket-versioning.go rename to api-bucket-versioning.go index 01458e85d..c521e4870 100644 --- a/api-get-bucket-versioning.go +++ b/api-bucket-versioning.go @@ -17,15 +17,65 @@ package minio import ( + "bytes" "context" "encoding/xml" - "io/ioutil" "net/http" "net/url" "github.com/minio/minio-go/v7/pkg/s3utils" ) +// SetBucketVersioning sets a bucket versioning configuration +func (c Client) SetBucketVersioning(ctx context.Context, bucketName string, config BucketVersioningConfiguration) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + buf, err := xml.Marshal(config) + if err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("versioning", "") + + reqMetadata := requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: bytes.NewReader(buf), + contentLength: int64(len(buf)), + contentMD5Base64: sumMD5Base64(buf), + contentSHA256Hex: sum256Hex(buf), + } + + // Execute PUT to set a bucket versioning. + resp, err := c.executeMethod(ctx, "PUT", reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, bucketName, "") + } + } + return nil +} + +// EnableVersioning - enable object versioning in given bucket. +func (c Client) EnableVersioning(ctx context.Context, bucketName string) error { + return c.SetBucketVersioning(ctx, bucketName, BucketVersioningConfiguration{Status: "Enabled"}) +} + +// SuspendVersioning - suspend object versioning in given bucket. +func (c Client) SuspendVersioning(ctx context.Context, bucketName string) error { + return c.SetBucketVersioning(ctx, bucketName, BucketVersioningConfiguration{Status: "Suspended"}) +} + // BucketVersioningConfiguration is the versioning configuration structure type BucketVersioningConfiguration struct { XMLName xml.Name `xml:"VersioningConfiguration"` @@ -61,14 +111,10 @@ func (c Client) GetBucketVersioning(ctx context.Context, bucketName string) (Buc return BucketVersioningConfiguration{}, httpRespToErrorResponse(resp, bucketName, "") } - bucketVersioningBuf, err := ioutil.ReadAll(resp.Body) - if err != nil { - return BucketVersioningConfiguration{}, err - } - versioningConfig := BucketVersioningConfiguration{} - if err := xml.Unmarshal(bucketVersioningBuf, &versioningConfig); err != nil { - return BucketVersioningConfiguration{}, err + if err = xmlDecoder(resp.Body, &versioningConfig); err != nil { + return versioningConfig, err } + return versioningConfig, nil } diff --git a/api-get-bucket-encryption.go b/api-get-bucket-encryption.go deleted file mode 100644 index 3f85668d0..000000000 --- a/api-get-bucket-encryption.go +++ /dev/null @@ -1,67 +0,0 @@ -/* - * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2020 MinIO, Inc. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "context" - "encoding/xml" - "io/ioutil" - "net/http" - "net/url" - - "github.com/minio/minio-go/v7/pkg/s3utils" -) - -// GetBucketEncryption gets the default encryption configuration -// on an existing bucket with a context to control cancellations and timeouts. -func (c Client) GetBucketEncryption(ctx context.Context, bucketName string) (ServerSideEncryptionConfiguration, error) { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return ServerSideEncryptionConfiguration{}, err - } - - // Get resources properly escaped and lined up before - // using them in http request. - urlValues := make(url.Values) - urlValues.Set("encryption", "") - - // Execute GET on bucket to get the default encryption configuration. - resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - }) - - defer closeResponse(resp) - if err != nil { - return ServerSideEncryptionConfiguration{}, err - } - - if resp.StatusCode != http.StatusOK { - return ServerSideEncryptionConfiguration{}, httpRespToErrorResponse(resp, bucketName, "") - } - - bucketEncryptionBuf, err := ioutil.ReadAll(resp.Body) - if err != nil { - return ServerSideEncryptionConfiguration{}, err - } - - encryptionConfig := ServerSideEncryptionConfiguration{} - if err := xml.Unmarshal(bucketEncryptionBuf, &encryptionConfig); err != nil { - return ServerSideEncryptionConfiguration{}, err - } - return encryptionConfig, nil -} diff --git a/api-get-bucket-replication.go b/api-get-bucket-replication.go deleted file mode 100644 index 37621f783..000000000 --- a/api-get-bucket-replication.go +++ /dev/null @@ -1,81 +0,0 @@ -/* - * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2020 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "context" - "encoding/xml" - "io/ioutil" - "net/http" - "net/url" - - "github.com/minio/minio-go/v7/pkg/replication" - "github.com/minio/minio-go/v7/pkg/s3utils" -) - -// GetBucketReplication fetches bucket replication configuration.If config is not -// found, returns empty config with nil error. -func (c Client) GetBucketReplication(ctx context.Context, bucketName string, opts ReplicationReqOptions) (cfg replication.Config, err error) { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return cfg, err - } - bucketReplicationCfg, err := c.getBucketReplication(ctx, bucketName) - if err != nil { - errResponse := ToErrorResponse(err) - if errResponse.Code == "ReplicationConfigurationNotFoundError" { - return cfg, nil - } - return cfg, err - } - return bucketReplicationCfg, nil -} - -// Request server for current bucket replication config. -func (c Client) getBucketReplication(ctx context.Context, bucketName string) (cfg replication.Config, err error) { - // Get resources properly escaped and lined up before - // using them in http request. - urlValues := make(url.Values) - urlValues.Set("replication", "") - - // Execute GET on bucket to get replication config. - resp, err := c.executeMethod(ctx, "GET", requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - }) - - defer closeResponse(resp) - if err != nil { - return cfg, err - } - - if resp.StatusCode != http.StatusOK { - return cfg, httpRespToErrorResponse(resp, bucketName, "") - } - - bucketReplicationBuf, err := ioutil.ReadAll(resp.Body) - if err != nil { - return cfg, err - } - replicationCfg := replication.Config{} - if err := xml.Unmarshal(bucketReplicationBuf, &replicationCfg); err != nil { - return cfg, err - } - - return replicationCfg, err -} diff --git a/api-get-lifecycle.go b/api-get-lifecycle.go deleted file mode 100644 index ff349f24a..000000000 --- a/api-get-lifecycle.go +++ /dev/null @@ -1,77 +0,0 @@ -/* - * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "context" - "io/ioutil" - "net/http" - "net/url" - - "github.com/minio/minio-go/v7/pkg/s3utils" -) - -// GetBucketLifecycle fetch bucket lifecycle configuration -func (c Client) GetBucketLifecycle(ctx context.Context, bucketName string) (string, error) { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return "", err - } - bucketLifecycle, err := c.getBucketLifecycle(ctx, bucketName) - if err != nil { - errResponse := ToErrorResponse(err) - if errResponse.Code == "NoSuchLifecycleConfiguration" { - return "", nil - } - return "", err - } - return bucketLifecycle, nil -} - -// Request server for current bucket lifecycle. -func (c Client) getBucketLifecycle(ctx context.Context, bucketName string) (string, error) { - // Get resources properly escaped and lined up before - // using them in http request. - urlValues := make(url.Values) - urlValues.Set("lifecycle", "") - - // Execute GET on bucket to get lifecycle. - resp, err := c.executeMethod(ctx, "GET", requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - }) - - defer closeResponse(resp) - if err != nil { - return "", err - } - - if resp != nil { - if resp.StatusCode != http.StatusOK { - return "", httpRespToErrorResponse(resp, bucketName, "") - } - } - - bucketLifecycleBuf, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", err - } - - lifecycle := string(bucketLifecycleBuf) - return lifecycle, err -} diff --git a/api-put-bucket.go b/api-put-bucket.go index 295a9a45d..d8b644328 100644 --- a/api-put-bucket.go +++ b/api-put-bucket.go @@ -21,36 +21,12 @@ import ( "bytes" "context" "encoding/xml" - "io/ioutil" "net/http" - "net/url" - "strings" - "github.com/minio/minio-go/v7/pkg/replication" "github.com/minio/minio-go/v7/pkg/s3utils" ) -// ApplyServerSideEncryptionByDefault defines default encryption configuration, KMS or SSE. To activate -// KMS, SSEAlgoritm needs to be set to "aws:kms" -// Minio currently does not support Kms. -type ApplyServerSideEncryptionByDefault struct { - KmsMasterKeyID string `xml:"KMSMasterKeyID,omitempty"` - SSEAlgorithm string `xml:"SSEAlgorithm"` -} - -// Rule layer encapsulates default encryption configuration -type Rule struct { - Apply ApplyServerSideEncryptionByDefault `xml:"ApplyServerSideEncryptionByDefault"` -} - -// ServerSideEncryptionConfiguration is the default encryption configuration structure -type ServerSideEncryptionConfiguration struct { - XMLName xml.Name `xml:"ServerSideEncryptionConfiguration"` - Rules []Rule `xml:"Rule"` -} - /// Bucket operations - func (c Client) makeBucket(ctx context.Context, bucketName string, opts MakeBucketOptions) (err error) { // Validate the input arguments. if err := s3utils.CheckValidBucketNameStrict(bucketName); err != nil { @@ -145,422 +121,3 @@ type MakeBucketOptions struct { func (c Client) MakeBucket(ctx context.Context, bucketName string, opts MakeBucketOptions) (err error) { return c.makeBucket(ctx, bucketName, opts) } - -// SetBucketPolicy sets the access permissions on an existing bucket. -func (c Client) SetBucketPolicy(ctx context.Context, bucketName, policy string) error { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return err - } - - // If policy is empty then delete the bucket policy. - if policy == "" { - return c.removeBucketPolicy(ctx, bucketName) - } - - // Save the updated policies. - return c.putBucketPolicy(ctx, bucketName, policy) -} - -// Saves a new bucket policy. -func (c Client) putBucketPolicy(ctx context.Context, bucketName, policy string) error { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return err - } - - // Get resources properly escaped and lined up before - // using them in http request. - urlValues := make(url.Values) - urlValues.Set("policy", "") - - // Content-length is mandatory for put policy request - policyReader := strings.NewReader(policy) - b, err := ioutil.ReadAll(policyReader) - if err != nil { - return err - } - - reqMetadata := requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - contentBody: policyReader, - contentLength: int64(len(b)), - } - - // Execute PUT to upload a new bucket policy. - resp, err := c.executeMethod(ctx, "PUT", reqMetadata) - defer closeResponse(resp) - if err != nil { - return err - } - if resp != nil { - if resp.StatusCode != http.StatusNoContent { - return httpRespToErrorResponse(resp, bucketName, "") - } - } - return nil -} - -// Removes all policies on a bucket. -func (c Client) removeBucketPolicy(ctx context.Context, bucketName string) error { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return err - } - // Get resources properly escaped and lined up before - // using them in http request. - urlValues := make(url.Values) - urlValues.Set("policy", "") - - // Execute DELETE on objectName. - resp, err := c.executeMethod(ctx, "DELETE", requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - contentSHA256Hex: emptySHA256Hex, - }) - defer closeResponse(resp) - if err != nil { - return err - } - return nil -} - -// SetBucketLifecycle set the lifecycle on an existing bucket. -func (c Client) SetBucketLifecycle(ctx context.Context, bucketName, lifecycle string) error { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return err - } - - // If lifecycle is empty then delete it. - if lifecycle == "" { - return c.removeBucketLifecycle(ctx, bucketName) - } - - // Save the updated lifecycle. - return c.putBucketLifecycle(ctx, bucketName, lifecycle) -} - -// Saves a new bucket lifecycle. -func (c Client) putBucketLifecycle(ctx context.Context, bucketName, lifecycle string) error { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return err - } - - // Get resources properly escaped and lined up before - // using them in http request. - urlValues := make(url.Values) - urlValues.Set("lifecycle", "") - - // Content-length is mandatory for put lifecycle request - lifecycleReader := strings.NewReader(lifecycle) - b, err := ioutil.ReadAll(lifecycleReader) - if err != nil { - return err - } - - reqMetadata := requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - contentBody: lifecycleReader, - contentLength: int64(len(b)), - contentMD5Base64: sumMD5Base64(b), - } - - // Execute PUT to upload a new bucket lifecycle. - resp, err := c.executeMethod(ctx, "PUT", reqMetadata) - defer closeResponse(resp) - if err != nil { - return err - } - if resp != nil { - if resp.StatusCode != http.StatusOK { - return httpRespToErrorResponse(resp, bucketName, "") - } - } - return nil -} - -// Remove lifecycle from a bucket. -func (c Client) removeBucketLifecycle(ctx context.Context, bucketName string) error { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return err - } - // Get resources properly escaped and lined up before - // using them in http request. - urlValues := make(url.Values) - urlValues.Set("lifecycle", "") - - // Execute DELETE on objectName. - resp, err := c.executeMethod(ctx, "DELETE", requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - contentSHA256Hex: emptySHA256Hex, - }) - defer closeResponse(resp) - if err != nil { - return err - } - return nil -} - -// SetBucketEncryption sets the default encryption configuration on an existing bucket. -func (c Client) SetBucketEncryption(ctx context.Context, bucketName string, configuration ServerSideEncryptionConfiguration) error { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return err - } - - buf, err := xml.Marshal(&configuration) - if err != nil { - return err - } - - // Get resources properly escaped and lined up before - // using them in http request. - urlValues := make(url.Values) - urlValues.Set("encryption", "") - - // Content-length is mandatory to set a default encryption configuration - reqMetadata := requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - contentBody: bytes.NewReader(buf), - contentLength: int64(len(buf)), - contentMD5Base64: sumMD5Base64(buf), - } - - // Execute PUT to upload a new bucket default encryption configuration. - resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) - defer closeResponse(resp) - if err != nil { - return err - } - if resp.StatusCode != http.StatusOK { - return httpRespToErrorResponse(resp, bucketName, "") - } - return nil -} - -// DeleteBucketEncryption removes the default encryption configuration on a bucket with a context to control cancellations and timeouts. -func (c Client) DeleteBucketEncryption(ctx context.Context, bucketName string) error { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return err - } - - // Get resources properly escaped and lined up before - // using them in http request. - urlValues := make(url.Values) - urlValues.Set("encryption", "") - - // DELETE default encryption configuration on a bucket. - resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - contentSHA256Hex: emptySHA256Hex, - }) - defer closeResponse(resp) - if err != nil { - return err - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { - return httpRespToErrorResponse(resp, bucketName, "") - } - return nil -} - -// SetBucketNotification saves a new bucket notification with a context to control cancellations -// and timeouts. -func (c Client) SetBucketNotification(ctx context.Context, bucketName string, bucketNotification BucketNotification) error { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return err - } - - // Get resources properly escaped and lined up before - // using them in http request. - urlValues := make(url.Values) - urlValues.Set("notification", "") - - notifBytes, err := xml.Marshal(bucketNotification) - if err != nil { - return err - } - - notifBuffer := bytes.NewReader(notifBytes) - reqMetadata := requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - contentBody: notifBuffer, - contentLength: int64(len(notifBytes)), - contentMD5Base64: sumMD5Base64(notifBytes), - contentSHA256Hex: sum256Hex(notifBytes), - } - - // Execute PUT to upload a new bucket notification. - resp, err := c.executeMethod(ctx, "PUT", reqMetadata) - defer closeResponse(resp) - if err != nil { - return err - } - if resp != nil { - if resp.StatusCode != http.StatusOK { - return httpRespToErrorResponse(resp, bucketName, "") - } - } - return nil -} - -// RemoveAllBucketNotification - Remove bucket notification clears all previously specified config -func (c Client) RemoveAllBucketNotification(ctx context.Context, bucketName string) error { - return c.SetBucketNotification(ctx, bucketName, BucketNotification{}) -} - -var ( - versionEnableConfig = []byte("Enabled") - versionEnableConfigLen = int64(len(versionEnableConfig)) - versionEnableConfigMD5Sum = sumMD5Base64(versionEnableConfig) - versionEnableConfigSHA256 = sum256Hex(versionEnableConfig) - - versionDisableConfig = []byte("Suspended") - versionDisableConfigLen = int64(len(versionDisableConfig)) - versionDisableConfigMD5Sum = sumMD5Base64(versionDisableConfig) - versionDisableConfigSHA256 = sum256Hex(versionDisableConfig) -) - -func (c Client) setVersioning(ctx context.Context, bucketName string, config []byte, length int64, md5sum, sha256sum string) error { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return err - } - - // Get resources properly escaped and lined up before - // using them in http request. - urlValues := make(url.Values) - urlValues.Set("versioning", "") - - reqMetadata := requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - contentBody: bytes.NewReader(config), - contentLength: length, - contentMD5Base64: md5sum, - contentSHA256Hex: sha256sum, - } - - // Execute PUT to set a bucket versioning. - resp, err := c.executeMethod(ctx, "PUT", reqMetadata) - defer closeResponse(resp) - if err != nil { - return err - } - if resp != nil { - if resp.StatusCode != http.StatusOK { - return httpRespToErrorResponse(resp, bucketName, "") - } - } - return nil -} - -// EnableVersioning - Enable object versioning in given bucket. -func (c Client) EnableVersioning(ctx context.Context, bucketName string) error { - return c.setVersioning(ctx, bucketName, versionEnableConfig, versionEnableConfigLen, versionEnableConfigMD5Sum, versionEnableConfigSHA256) -} - -// DisableVersioning - Disable object versioning in given bucket. -func (c Client) DisableVersioning(ctx context.Context, bucketName string) error { - return c.setVersioning(ctx, bucketName, versionDisableConfig, versionDisableConfigLen, versionDisableConfigMD5Sum, versionDisableConfigSHA256) -} - -// ReplicationReqOptions represents options specified by user for Replication call. Options may be added in the future. -type ReplicationReqOptions struct{} - -// RemoveBucketReplication removes a replication config on an existing bucket. -func (c Client) RemoveBucketReplication(ctx context.Context, bucketName string, opts ReplicationReqOptions) error { - return c.removeBucketReplication(ctx, bucketName) -} - -// SetBucketReplication sets a replication config on an existing bucket. -func (c Client) SetBucketReplication(ctx context.Context, bucketName string, cfg replication.Config, opts ReplicationReqOptions) error { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return err - } - - // If replication is empty then delete it. - if cfg.Empty() { - return c.removeBucketReplication(ctx, bucketName) - } - // Save the updated replication. - return c.putBucketReplication(ctx, bucketName, cfg) -} - -// Saves a new bucket replication. -func (c Client) putBucketReplication(ctx context.Context, bucketName string, cfg replication.Config) error { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return err - } - - // Get resources properly escaped and lined up before - // using them in http request. - urlValues := make(url.Values) - urlValues.Set("replication", "") - replication, err := xml.Marshal(cfg) - if err != nil { - return err - } - // Content-length is mandatory for put replication request - replicationReader := bytes.NewReader(replication) - b, err := ioutil.ReadAll(replicationReader) - if err != nil { - return err - } - reqMetadata := requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - contentBody: replicationReader, - contentLength: int64(len(b)), - contentMD5Base64: sumMD5Base64(b), - } - // Execute PUT to upload a new bucket replication config. - resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) - defer closeResponse(resp) - if err != nil { - return err - } - - if resp.StatusCode != http.StatusOK { - return httpRespToErrorResponse(resp, bucketName, "") - } - - return nil -} - -// Remove replication from a bucket. -func (c Client) removeBucketReplication(ctx context.Context, bucketName string) error { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return err - } - // Get resources properly escaped and lined up before - // using them in http request. - urlValues := make(url.Values) - urlValues.Set("replication", "") - - // Execute DELETE on objectName. - resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - contentSHA256Hex: emptySHA256Hex, - }) - defer closeResponse(resp) - if err != nil { - return err - } - return nil -} diff --git a/docs/API.md b/docs/API.md index 3920088e3..6c7839b4a 100644 --- a/docs/API.md +++ b/docs/API.md @@ -50,29 +50,29 @@ func main() { } ``` -| Bucket operations | Object operations | Encrypted Object operations | Presigned operations | Bucket Policy/Notification Operations | Client custom settings | -| :--- | :--- | :--- | :--- | :--- | :--- | -| [`MakeBucket`](#MakeBucket) | [`GetObject`](#GetObject) | [`GetObject`](#GetObject) | [`PresignedGetObject`](#PresignedGetObject) | [`SetBucketPolicy`](#SetBucketPolicy) | [`SetAppInfo`](#SetAppInfo) | -| [`PutObject`](#PutObject) | [`PutObject`](#PutObject) | [`PresignedPutObject`](#PresignedPutObject) | [`GetBucketPolicy`](#GetBucketPolicy) | [`SetCustomTransport`](#SetCustomTransport) | -| [`ListBuckets`](#ListBuckets) | [`CopyObject`](#CopyObject) | [`CopyObject`](#CopyObject) | [`PresignedPostPolicy`](#PresignedPostPolicy) | [`SetBucketNotification`](#SetBucketNotification) | [`TraceOn`](#TraceOn) | -| [`BucketExists`](#BucketExists) | [`StatObject`](#StatObject) | [`StatObject`](#StatObject) | | [`GetBucketNotification`](#GetBucketNotification) | [`TraceOff`](#TraceOff) | -| [`RemoveBucket`](#RemoveBucket) | [`RemoveObject`](#RemoveObject) | [`FPutObject`](#FPutObject) | | [`RemoveAllBucketNotification`](#RemoveAllBucketNotification) | [`SetS3TransferAccelerate`](#SetS3TransferAccelerate) | -| [`ListObjects`](#ListObjects) | [`RemoveObjects`](#RemoveObjects) | [`FGetObject`](#FGetObject) | | [`ListenBucketNotification`](#ListenBucketNotification) | | -| | [`RemoveIncompleteUpload`](#RemoveIncompleteUpload) | [`ComposeObject`](#ComposeObjecet) | | [`SetBucketLifecycle`](#SetBucketLifecycle) | | -| [`ListIncompleteUploads`](#ListIncompleteUploads) | [`FPutObject`](#FPutObject) | | | [`GetBucketLifecycle`](#GetBucketLifecycle) | | -| [`SetBucketTagging`](#SetBucketTagging) | [`FGetObject`](#FGetObject) | | | [`SetObjectLockConfig`](#SetObjectLockConfig) | | -| [`GetBucketTagging`](#GetBucketTagging) | [`ComposeObject`](#ComposeObject) | | | [`GetObjectLockConfig`](#GetObjectLockConfig) | | -| [`DeleteBucketTagging`](#DeleteBucketTagging) | | | | [`EnableVersioning`](#EnableVersioning) | | -| [`SetBucketReplication`](#SetBucketReplication) | | | | [`DisableVersioning`](#DisableVersioning) | | -| [`GetBucketReplication`](#GetBucketReplication) | [`PutObjectRetention`](#PutObjectRetention) | | | [`GetBucketEncryption`](#GetBucketEncryption) | | -| [`RemoveBucketReplication`](#RemoveBucketReplication) | [`GetObjectRetention`](#GetObjectRetention) | | | [`DeleteBucketEncryption`](#DeleteBucketEncryption) | | -| | [`PutObjectLegalHold`](#PutObjectLegalHold) | | | | | -| | [`GetObjectLegalHold`](#GetObjectLegalHold) | | | | | -| | [`SelectObjectContent`](#SelectObjectContent) | | | | | -| | [`PutObjectTagging`](#PutObjectTagging) | | | | | -| | [`GetObjectTagging`](#GetObjectTagging) | | | | | -| | [`RemoveObjectTagging`](#RemoveObjectTagging) | | | | | -| | | | | | | +| Bucket operations | Object operations | Encrypted Object operations | Presigned operations | Bucket Policy/Notification Operations | Client custom settings | +| :--- | :--- | :--- | :--- | :--- | :--- | +| [`MakeBucket`](#MakeBucket) | [`GetObject`](#GetObject) | [`GetObject`](#GetObject) | [`PresignedGetObject`](#PresignedGetObject) | [`SetBucketPolicy`](#SetBucketPolicy) | [`SetAppInfo`](#SetAppInfo) | +| [`PutObject`](#PutObject) | [`PutObject`](#PutObject) | [`PresignedPutObject`](#PresignedPutObject) | [`GetBucketPolicy`](#GetBucketPolicy) | [`SetCustomTransport`](#SetCustomTransport) | | +| [`ListBuckets`](#ListBuckets) | [`CopyObject`](#CopyObject) | [`CopyObject`](#CopyObject) | [`PresignedPostPolicy`](#PresignedPostPolicy) | [`SetBucketNotification`](#SetBucketNotification) | [`TraceOn`](#TraceOn) | +| [`BucketExists`](#BucketExists) | [`StatObject`](#StatObject) | [`StatObject`](#StatObject) | | [`GetBucketNotification`](#GetBucketNotification) | [`TraceOff`](#TraceOff) | +| [`RemoveBucket`](#RemoveBucket) | [`RemoveObject`](#RemoveObject) | [`FPutObject`](#FPutObject) | | [`RemoveAllBucketNotification`](#RemoveAllBucketNotification) | [`SetS3TransferAccelerate`](#SetS3TransferAccelerate) | +| [`ListObjects`](#ListObjects) | [`RemoveObjects`](#RemoveObjects) | [`FGetObject`](#FGetObject) | | [`ListenBucketNotification`](#ListenBucketNotification) | | +| | [`RemoveIncompleteUpload`](#RemoveIncompleteUpload) | [`ComposeObject`](#ComposeObjecet) | | [`SetBucketLifecycle`](#SetBucketLifecycle) | | +| [`ListIncompleteUploads`](#ListIncompleteUploads) | [`FPutObject`](#FPutObject) | | | [`GetBucketLifecycle`](#GetBucketLifecycle) | | +| [`SetBucketTagging`](#SetBucketTagging) | [`FGetObject`](#FGetObject) | | | [`SetObjectLockConfig`](#SetObjectLockConfig) | | +| [`GetBucketTagging`](#GetBucketTagging) | [`ComposeObject`](#ComposeObject) | | | [`GetObjectLockConfig`](#GetObjectLockConfig) | | +| [`RemoveBucketTagging`](#RemoveBucketTagging) | | | | [`EnableVersioning`](#EnableVersioning) | | +| [`SetBucketReplication`](#SetBucketReplication) | | | | [`DisableVersioning`](#DisableVersioning) | | +| [`GetBucketReplication`](#GetBucketReplication) | [`PutObjectRetention`](#PutObjectRetention) | | | [`GetBucketEncryption`](#GetBucketEncryption) | | +| [`RemoveBucketReplication`](#RemoveBucketReplication) | [`GetObjectRetention`](#GetObjectRetention) | | | [`RemoveBucketEncryption`](#RemoveBucketEncryption) | | +| | [`PutObjectLegalHold`](#PutObjectLegalHold) | | | | | +| | [`GetObjectLegalHold`](#GetObjectLegalHold) | | | | | +| | [`SelectObjectContent`](#SelectObjectContent) | | | | | +| | [`PutObjectTagging`](#PutObjectTagging) | | | | | +| | [`GetObjectTagging`](#GetObjectTagging) | | | | | +| | [`RemoveObjectTagging`](#RemoveObjectTagging) | | | | | +| | | | | | | ## 1. Constructor @@ -404,9 +404,9 @@ if err != nil { fmt.Printf("Fetched Object Tags: %v\n", tags) ``` - -### DeleteBucketTagging(ctx context.Context, bucketName string) error -Deletes all tags of a bucket. + +### RemoveBucketTagging(ctx context.Context, bucketName string) error +Removes all tags on a bucket. __Parameters__ @@ -417,7 +417,7 @@ __Parameters__ __Example__ ```go -err := minioClient.DeleteBucketTagging(context.Background(), "my-bucketname") +err := minioClient.RemoveBucketTagging(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } @@ -1341,7 +1341,7 @@ if err != nil { ``` -### GetBucketNotification(ctx context.Context, bucketName string) (BucketNotification, error) +### GetBucketNotification(ctx context.Context, bucketName string) (notification.Configuration, error) Get notification configuration on a bucket. __Parameters__ @@ -1357,7 +1357,7 @@ __Return Values__ |Param |Type |Description | |:---|:---| :---| -|`bucketNotification` | _minio.BucketNotification_ |structure which holds all notification configurations| +|`config` | _notification.Configuration_ |structure which holds all notification configurations| |`err` | _error_ |Standard Error | __Example__ @@ -1378,7 +1378,7 @@ for _, queueConfig := range bucketNotification.QueueConfigs { ``` -### SetBucketNotification(ctx context.Context, bucketName string, bucketNotification BucketNotification) error +### SetBucketNotification(ctx context.Context, bucketName string, config notification.Configuration) error Set a new bucket notification on a bucket. __Parameters__ @@ -1388,7 +1388,7 @@ __Parameters__ |:---|:---| :---| |`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | -|`bucketNotification` | _minio.BucketNotification_ |Represents the XML to be sent to the configured web service | +|`config` | _notification.Configuration_ |Represents the XML to be sent to the configured web service | __Return Values__ @@ -1401,17 +1401,17 @@ __Example__ ```go -queueArn := minio.NewArn("aws", "sqs", "us-east-1", "804605494417", "PhotoUpdate") +queueArn := notification.NewArn("aws", "sqs", "us-east-1", "804605494417", "PhotoUpdate") -queueConfig := minio.NewNotificationConfig(queueArn) +queueConfig := notification.NewConfig(queueArn) queueConfig.AddEvents(minio.ObjectCreatedAll, minio.ObjectRemovedAll) queueConfig.AddFilterPrefix("photos/") queueConfig.AddFilterSuffix(".jpg") -bucketNotification := minio.BucketNotification{} -bucketNotification.AddQueue(queueConfig) +config := notification.Configuration{} +config.AddQueue(queueConfig) -err = minioClient.SetBucketNotification(context.Background(), "mybucket", bucketNotification) +err = minioClient.SetBucketNotification(context.Background(), "mybucket", config) if err != nil { fmt.Println("Unable to set the bucket notification: ", err) return @@ -1449,7 +1449,7 @@ if err != nil { ``` -### ListenBucketNotification(context context.Context, bucketName, prefix, suffix string, events []string) <-chan NotificationInfo +### ListenBucketNotification(context context.Context, bucketName, prefix, suffix string, events []string) <-chan notification.Info ListenBucketNotification API receives bucket notification events through the notification channel. The returned notification channel has two fields 'Records' and 'Err'. - 'Records' holds the notifications received from the server. @@ -1471,12 +1471,12 @@ __Return Values__ |Param |Type |Description | |:---|:---| :---| -|`notificationInfo` | _chan minio.NotificationInfo_ | Channel of bucket notifications | +|`notificationInfo` | _chan notification.Info_ | Channel of bucket notifications | __minio.NotificationInfo__ |Field |Type |Description | -|`notificationInfo.Records` | _[]minio.NotificationEvent_ | Collection of notification events | +|`notificationInfo.Records` | _[]notification.Event_ | Collection of notification events | |`notificationInfo.Err` | _error_ | Carries any error occurred during the operation (Standard Error) | @@ -1498,7 +1498,7 @@ for notificationInfo := range minioClient.ListenBucketNotification(context.Backg ``` -### SetBucketLifecycle(ctx context.Context, bucketname, lifecycle string) error +### SetBucketLifecycle(ctx context.Context, bucketname, config *lifecycle.Configuration) error Set lifecycle on bucket or an object prefix. __Parameters__ @@ -1507,7 +1507,7 @@ __Parameters__ |:---|:---| :---| |`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket| -|`lifecycle` | _string_ |Lifecycle to be set | +|`config` | _lifecycle.Configuration_ |Lifecycle to be set | __Return Values__ @@ -1518,18 +1518,16 @@ __Return Values__ __Example__ ```go -lifecycle := ` - - expire-bucket - - Enabled - - 365 - - -` - -err = minioClient.SetBucketLifecycle(context.Background(), "my-bucketname", lifecycle) +config := lifecycle.NewConfiguration() +config.Rules = []lifecycle.Rule{ + ID: "expire-bucket", + Status: "Enabled", + Expiration: lifecycle.Expiration{ + Days: 365, + }, +} + +err = minioClient.SetBucketLifecycle(context.Background(), "my-bucketname", config) if err != nil { fmt.Println(err) return @@ -1537,7 +1535,7 @@ if err != nil { ``` -### GetBucketLifecycle(ctx context.Context, bucketName string) (lifecycle string, error) +### GetBucketLifecycle(ctx context.Context, bucketName string) (*lifecycle.Configuration error) Get lifecycle on a bucket or a prefix. __Parameters__ @@ -1553,7 +1551,7 @@ __Return Values__ |Param |Type |Description | |:---|:---| :---| -|`lifecycle` | _string_ |Lifecycle returned from the server | +|`config` | _lifecycle.Configuration_ |Lifecycle returned from the server | |`err` | _error_ |Standard Error | __Example__ @@ -1566,7 +1564,7 @@ if err != nil { ``` -### SetBucketEncryption(ctx context.Context, bucketname string, configuration ServerSideEncryptionConfiguration) error +### SetBucketEncryption(ctx context.Context, bucketname string, config sse.Configuration) error Set default encryption configuration on a bucket. __Parameters__ @@ -1575,7 +1573,7 @@ __Parameters__ |:---|:---| :---| |`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket| -|`configuration` | _minio.ServerSideEncyrptionConfiguration_ | Structure that holds default encryption configuration to be set | +|`config` | _sse.Configuration_ | Structure that holds default encryption configuration to be set | __Return Values__ @@ -1591,23 +1589,15 @@ if err != nil { log.Fatalln(err) } -// Initialize default encryption configuration structure -config := minio.ServerSideEncryptionConfiguration{Rules: []minio.Rule{ - minio.Rule{ - Apply: minio.ApplyServerSideEncryptionByDefault{ - SSEAlgorithm: "AES256", - }, - }, -}} // Set default encryption configuration on an S3 bucket -err = s3Client.SetBucketEncryption(context.Background(), "my-bucketname", config) +err = s3Client.SetBucketEncryption(context.Background(), "my-bucketname", sse.NewConfigurationSSES3()) if err != nil { log.Fatalln(err) } ``` -### GetBucketEncryption(ctx context.Context, bucketName string) (ServerSideEncryptionConfiguration, error) +### GetBucketEncryption(ctx context.Context, bucketName string) (*sse.Configuration, error) Get default encryption configuration set on a bucket. __Parameters__ @@ -1623,7 +1613,7 @@ __Return Values__ |Param |Type |Description | |:---|:---| :---| -|`configuration` | _minio.ServerSideEncyrptionConfiguration_ | Structure that holds default encryption configuration | +|`config` | _sse.Configuration_ | Structure that holds default encryption configuration | |`err` | _error_ |Standard Error | __Example__ @@ -1642,9 +1632,9 @@ if err != nil { fmt.Printf("%+v\n", encryptionConfig) ``` - -### DeleteBucketEncryption(ctx context.Context, bucketName string) (error) -Delete/Remove default encryption configuration set on a bucket. + +### RemoveBucketEncryption(ctx context.Context, bucketName string) (error) +Remove default encryption configuration set on a bucket. __Parameters__ @@ -1664,7 +1654,7 @@ __Return Values__ __Example__ ```go -err := s3Client.DeleteBucketEncryption(context.Background(), "my-bucketname") +err := s3Client.RemoveBucketEncryption(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } @@ -1841,9 +1831,8 @@ fmt.Printf("%+v\n", versioningConfig) -### SetBucketReplication(ctx context.Context, bucketname, cfg replication.Config, opts ReplicationReqOptions) error -Set replication configuration on a bucket. To use this API with MinIO server, ReplicationArn should be set in the replication config. Replication ARN can be obtained by first defining the replication target on MinIO -using `mc admin bucket replication set` to associate the source and destination buckets for replication with the replication endpoint. Next, issue a `mc admin bucket remote` to fetch the replication ARN associated with this replication endpoint. +### SetBucketReplication(ctx context.Context, bucketname, cfg replication.Config) error +Set replication configuration on a bucket. To use this API with MinIO server, ReplicationArn should be set in the replication config. Replication ARN can be obtained by first defining the replication target on MinIO using `mc admin bucket replication set` to associate the source and destination buckets for replication with the replication endpoint. Next, issue a `mc admin bucket remote` to fetch the replication ARN associated with this replication endpoint. __Parameters__ @@ -1852,7 +1841,6 @@ __Parameters__ |`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket| |`cfg` | _replication.Config_ |Replication configuration to be set | -|`opts` | _ReplicationReqOptions_ |Options for replication - This is currently empty.| __Return Values__ @@ -1900,7 +1888,7 @@ if err := xml.Unmarshal([]byte(replicationStr), &replicationConfig); err != nil } // this is optional for replication with MinIO server. cfg.ReplicationArn := "arn:minio:s3::598361bf-3cec-49a7-b529-ce870a34d759:*" -err = minioClient.SetBucketReplication(context.Background(), "my-bucketname", replicationConfig, ReplicationReqOptions{}) +err = minioClient.SetBucketReplication(context.Background(), "my-bucketname", replicationConfig) if err != nil { fmt.Println(err) return @@ -1908,8 +1896,7 @@ if err != nil { ``` - -### GetBucketReplication(ctx context.Context, bucketName string, opts ReplicationReqOptions) (replication.Config, error) +### GetBucketReplication(ctx context.Context, bucketName string) (replication.Config, error) Get current replication config on a bucket. __Parameters__ @@ -1919,7 +1906,6 @@ __Parameters__ |:---|:---| :---| |`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | -|`opts` | _ReplicationReqOptions_ | options for replication request __Return Values__ @@ -1939,16 +1925,15 @@ if err != nil { ``` - -### RemoveBucketReplication(ctx context.Context, bucketname string, opts ReplicationReqOptions) error +### RemoveBucketReplication(ctx context.Context, bucketname string) error Removes replication configuration on a bucket. + __Parameters__ |Param |Type |Description | |:---|:---| :---| |`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket| -|`opts` | _ReplicationReqOptions_ | options for replication request __Return Values__ diff --git a/examples/s3/deletebucketencryption.go b/examples/s3/removebucketencryption.go similarity index 95% rename from examples/s3/deletebucketencryption.go rename to examples/s3/removebucketencryption.go index 7345fde8f..71f1926b1 100644 --- a/examples/s3/deletebucketencryption.go +++ b/examples/s3/removebucketencryption.go @@ -43,7 +43,7 @@ func main() { // s3Client.TraceOn(os.Stderr) // Get default encryption configuration set on a S3 bucket - err = s3Client.DeleteBucketEncryption(context.Background(), "my-bucketname") + err = s3Client.RemoveBucketEncryption(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } diff --git a/examples/s3/removebuckettagging.go b/examples/s3/removebuckettagging.go index b78528876..534ef7403 100644 --- a/examples/s3/removebuckettagging.go +++ b/examples/s3/removebuckettagging.go @@ -39,7 +39,7 @@ func main() { if err != nil { log.Fatalln(err) } - err = s3Client.DeleteBucketTagging(context.Background(), "my-bucketname") + err = s3Client.RemoveBucketTagging(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } diff --git a/examples/s3/setbucketencryption.go b/examples/s3/setbucketencryption.go index e3e39f8b7..b742f1e73 100644 --- a/examples/s3/setbucketencryption.go +++ b/examples/s3/setbucketencryption.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/sse" ) func main() { @@ -43,16 +44,7 @@ func main() { // s3Client.TraceOn(os.Stderr) // Set default encryption configuration on a bucket - config := minio.ServerSideEncryptionConfiguration{Rules: []minio.Rule{ - { - Apply: minio.ApplyServerSideEncryptionByDefault{ - SSEAlgorithm: "AES256", - // KmsMasterKeyID: "my-masterkey", - // SSEAlgorithm: "aws:kms", - }, - }, - }} - err = s3Client.SetBucketEncryption(context.Background(), "my-bucketname", config) + err = s3Client.SetBucketEncryption(context.Background(), "my-bucketname", sse.NewConfigurationSSES3()) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/setbucketlifecycle.go b/examples/s3/setbucketlifecycle.go index 927a060df..a527d797c 100644 --- a/examples/s3/setbucketlifecycle.go +++ b/examples/s3/setbucketlifecycle.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/lifecycle" ) func main() { @@ -43,8 +44,15 @@ func main() { // s3Client.TraceOn(os.Stderr) // Set lifecycle on a bucket - lifecycle := `expire-bucketEnabled365` - err = s3Client.SetBucketLifecycle(context.Background(), "my-bucketname", lifecycle) + config := lifecycle.NewConfiguration() + config.Rules = []lifecycle.Rule{ + ID: "expire-bucket", + Status: "Enabled", + Expiration: lifecycle.Expiration{ + Days: 365, + }, + } + err = s3Client.SetBucketLifecycle(context.Background(), "my-bucketname", config) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/setbucketnotification.go b/examples/s3/setbucketnotification.go index 5e3b434d3..a915fa4f9 100644 --- a/examples/s3/setbucketnotification.go +++ b/examples/s3/setbucketnotification.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/notification" ) func main() { @@ -56,30 +57,30 @@ func main() { // with actual values that you receive from the S3 provider // Here you create a new Topic notification - topicArn := minio.NewArn("YOUR-PROVIDER", "YOUR-SERVICE", "YOUR-REGION", "YOUR-ACCOUNT-ID", "YOUR-RESOURCE") - topicConfig := minio.NewNotificationConfig(topicArn) + topicArn := notification.NewArn("YOUR-PROVIDER", "YOUR-SERVICE", "YOUR-REGION", "YOUR-ACCOUNT-ID", "YOUR-RESOURCE") + topicConfig := notification.NewConfig(topicArn) topicConfig.AddEvents(minio.ObjectCreatedAll, minio.ObjectRemovedAll) topicConfig.AddFilterPrefix("photos/") topicConfig.AddFilterSuffix(".jpg") // Create a new Queue notification - queueArn := minio.NewArn("YOUR-PROVIDER", "YOUR-SERVICE", "YOUR-REGION", "YOUR-ACCOUNT-ID", "YOUR-RESOURCE") - queueConfig := minio.NewNotificationConfig(queueArn) + queueArn := notification.NewArn("YOUR-PROVIDER", "YOUR-SERVICE", "YOUR-REGION", "YOUR-ACCOUNT-ID", "YOUR-RESOURCE") + queueConfig := notification.NewConfig(queueArn) queueConfig.AddEvents(minio.ObjectRemovedAll) // Create a new Lambda (CloudFunction) - lambdaArn := minio.NewArn("YOUR-PROVIDER", "YOUR-SERVICE", "YOUR-REGION", "YOUR-ACCOUNT-ID", "YOUR-RESOURCE") - lambdaConfig := minio.NewNotificationConfig(lambdaArn) + lambdaArn := notification.NewArn("YOUR-PROVIDER", "YOUR-SERVICE", "YOUR-REGION", "YOUR-ACCOUNT-ID", "YOUR-RESOURCE") + lambdaConfig := notification.NewConfig(lambdaArn) lambdaConfig.AddEvents(minio.ObjectRemovedAll) lambdaConfig.AddFilterSuffix(".swp") // Now, set all previously created notification configs - bucketNotification := minio.BucketNotification{} - bucketNotification.AddTopic(topicConfig) - bucketNotification.AddQueue(queueConfig) - bucketNotification.AddLambda(lambdaConfig) + config := ¬ification.Configuration{} + config.AddTopic(topicConfig) + config.AddQueue(queueConfig) + config.AddLambda(lambdaConfig) - err = s3Client.SetBucketNotification(context.Background(), "YOUR-BUCKET", bucketNotification) + err = s3Client.SetBucketNotification(context.Background(), "YOUR-BUCKET", config) if err != nil { log.Fatalln("Error: " + err.Error()) } diff --git a/examples/s3/setbucketreplication.go b/examples/s3/setbucketreplication.go index 428e7dab0..3b668f77f 100644 --- a/examples/s3/setbucketreplication.go +++ b/examples/s3/setbucketreplication.go @@ -41,7 +41,9 @@ func main() { if err != nil { log.Fatalln(err) } + // s3Client.TraceOn(os.Stderr) + replicationStr := `stringEnabled1Disabledarn:aws:s3:::destPrefixTag-Key1Tag-Value1Tag-Key2Tag-Value2` var replCfg replication.Config err = xml.Unmarshal([]byte(replicationStr), &replCfg) @@ -50,9 +52,9 @@ func main() { } // This replication ARN should have been generated for replication endpoint using `mc admin bucket remote` command - replCfg.replicationArn = "arn:minio:s3::dadddae7-f1d7-440f-b5d6-651aa9a8c8a7:*" + replCfg.ReplicationARN = "arn:minio:s3::dadddae7-f1d7-440f-b5d6-651aa9a8c8a7:*" // Set replication config on a bucket - err = s3Client.SetBucketReplication(context.Background(), "my-bucketname", replCfg, ReplicationReqOptions{}) + err = s3Client.SetBucketReplication(context.Background(), "my-bucketname", replCfg) if err != nil { log.Fatalln(err) } diff --git a/functional_tests.go b/functional_tests.go index 4e487f6af..2a18e8a86 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -45,6 +45,7 @@ import ( "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/encrypt" + "github.com/minio/minio-go/v7/pkg/notification" ) const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" @@ -4985,51 +4986,50 @@ func testBucketNotification() { bucketName := os.Getenv("NOTIFY_BUCKET") args["bucketName"] = bucketName - topicArn := minio.NewArn("aws", os.Getenv("NOTIFY_SERVICE"), os.Getenv("NOTIFY_REGION"), os.Getenv("NOTIFY_ACCOUNTID"), os.Getenv("NOTIFY_RESOURCE")) - queueArn := minio.NewArn("aws", "dummy-service", "dummy-region", "dummy-accountid", "dummy-resource") + topicArn := notification.NewArn("aws", os.Getenv("NOTIFY_SERVICE"), os.Getenv("NOTIFY_REGION"), os.Getenv("NOTIFY_ACCOUNTID"), os.Getenv("NOTIFY_RESOURCE")) + queueArn := notification.NewArn("aws", "dummy-service", "dummy-region", "dummy-accountid", "dummy-resource") - topicConfig := minio.NewNotificationConfig(topicArn) - - topicConfig.AddEvents(minio.ObjectCreatedAll, minio.ObjectRemovedAll) + topicConfig := notification.NewConfig(topicArn) + topicConfig.AddEvents(notification.ObjectCreatedAll, notification.ObjectRemovedAll) topicConfig.AddFilterSuffix("jpg") - queueConfig := minio.NewNotificationConfig(queueArn) - queueConfig.AddEvents(minio.ObjectCreatedAll) + queueConfig := notification.NewConfig(queueArn) + queueConfig.AddEvents(notification.ObjectCreatedAll) queueConfig.AddFilterPrefix("photos/") - bNotification := minio.BucketNotification{} - bNotification.AddTopic(topicConfig) + config := notification.Configuration{} + config.AddTopic(topicConfig) // Add the same topicConfig again, should have no effect // because it is duplicated - bNotification.AddTopic(topicConfig) - if len(bNotification.TopicConfigs) != 1 { + config.AddTopic(topicConfig) + if len(config.TopicConfigs) != 1 { logError(testName, function, args, startTime, "", "Duplicate entry added", err) return } // Add and remove a queue config - bNotification.AddQueue(queueConfig) - bNotification.RemoveQueueByArn(queueArn) + config.AddQueue(queueConfig) + config.RemoveQueueByArn(queueArn) - err = c.SetBucketNotification(context.Background(), bucketName, bNotification) + err = c.SetBucketNotification(context.Background(), bucketName, config) if err != nil { logError(testName, function, args, startTime, "", "SetBucketNotification failed", err) return } - bNotification, err = c.GetBucketNotification(context.Background(), bucketName) + config, err = c.GetBucketNotification(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketNotification failed", err) return } - if len(bNotification.TopicConfigs) != 1 { + if len(config.TopicConfigs) != 1 { logError(testName, function, args, startTime, "", "Topic config is empty", err) return } - if bNotification.TopicConfigs[0].Filter.S3Key.FilterRules[0].Value != "jpg" { + if config.TopicConfigs[0].Filter.S3Key.FilterRules[0].Value != "jpg" { logError(testName, function, args, startTime, "", "Couldn't get the suffix", err) return } diff --git a/go.mod b/go.mod index 8b6e120a2..6d0c73708 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,13 @@ module github.com/minio/minio-go/v7 go 1.12 require ( + github.com/dustin/go-humanize v1.0.0 // indirect github.com/google/uuid v1.1.1 github.com/json-iterator/go v1.1.10 github.com/minio/md5-simd v1.1.0 github.com/minio/sha256-simd v0.1.1 github.com/mitchellh/go-homedir v1.1.0 + github.com/sirupsen/logrus v1.6.0 // indirect github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f golang.org/x/net v0.0.0-20190522155817-f3200d17e092 diff --git a/go.sum b/go.sum index 2406460b8..df4b74773 100644 --- a/go.sum +++ b/go.sum @@ -1,247 +1,57 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.39.0/go.mod h1:rVLT6fkc8chs9sfPtFc1SBH6em7n+ZoXaG+87tDISts= -contrib.go.opencensus.io/exporter/ocagent v0.5.0/go.mod h1:ImxhfLRpxoYiSq891pBrLVhN+qmP8BTVvdH2YLs7Gl0= -git.apache.org/thrift.git v0.13.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= -github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= -github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0= -github.com/Azure/go-autorest v11.7.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/Shopify/sarama v1.24.1/go.mod h1:fGP8eQ6PugKEI0iUETYYtnP6d1pH/bdDMTel1X5ajsU= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/alecthomas/participle v0.2.1/go.mod h1:SW6HZGeZgSIpcUWX3fXpfZhuaWHnmoD5KCVaqSaNTkk= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aws/aws-sdk-go v1.20.21/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2/go.mod h1:RDu/qcrnpEdJC/p8tx34+YBFqqX71lB7dOX9QE+ZC4M= -github.com/beevik/ntp v0.2.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= -github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cheggaaa/pb v1.0.28/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= -github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= -github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= -github.com/coredns/coredns v1.4.0/go.mod h1:zASH/MVDgR6XZTbxvOnsZfffS+31vg6Ackf/wo1+AM0= -github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.12+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= -github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.9.4/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.10.1/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/cpuid v1.2.2/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= -github.com/minio/simdjson-go v0.1.5-0.20200303142138-b17fe061ea37/go.mod h1:oKURrZZEBtqObgJrSjN1Ln2n9MJj2icuBTkeJzZnvSI= -github.com/minio/sio v0.2.0 h1:NCRCFLx0r5pRbXf65LVNjxbCGZgNQvNFQkgX3XF4BoA= -github.com/minio/sio v0.2.0/go.mod h1:nKM5GIWSrqbOZp0uhyj6M1iA0X6xQzSGtYSaTKSCut0= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/mmcloughlin/avo v0.0.0-20200303042253-6df701fe672f/go.mod h1:L0u9qfRMLNBO97u6pPukRp6ncoQz0Q25W69fvtht3vA= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/montanaflynn/stats v0.5.0 h1:2EkzeTSqBB4V4bJwWrt5gIIrZmpJBcoIRGS2kWLgzmk= -github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nats-io/gnatsd v1.4.1/go.mod h1:nqco77VO78hLCJpIcVfygDP2rPGfsEHkGTUk94uh5DQ= -github.com/nats-io/go-nats v1.7.2/go.mod h1:+t7RHT5ApZebkrQdnn6AhQJmhJJiKAvJUio1PiiCtj0= -github.com/nats-io/go-nats-streaming v0.4.4/go.mod h1:gfq4R3c9sKAINOpelo0gn/b9QDMBZnmrttcsNF+lqyo= -github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= -github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= -github.com/nats-io/nats-server v1.4.1/go.mod h1:c8f/fHd2B6Hgms3LtCaI7y6pC4WD1f4SUxcCud5vhBc= -github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= -github.com/nats-io/nats-streaming-server v0.14.2/go.mod h1:RyqtDJZvMZO66YmyjIYdIvS69zu/wDAkyNWa8PIUa5c= -github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= -github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nats-io/stan.go v0.4.5/go.mod h1:Ji7mK6gRZJSH1nc3ZJH6vi7zn/QnZhpR9Arm4iuzsUQ= -github.com/ncw/directio v1.0.5 h1:JSUBhdjEvVaJvOoyPAbcW0fnd0tvRXD76wEfZ1KcQz4= -github.com/ncw/directio v1.0.5/go.mod h1:rX/pKEYkOXBGOggmcyJeJGloCkleSvphPx2eV3t6ROk= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nsqio/go-nsq v1.0.7/go.mod h1:XP5zaUs3pqf+Q71EqUJs3HYfBIqfK6G83WQMdNN+Ito= -github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= -github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4 v2.2.6+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4 v2.4.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/tidwall/gjson v1.3.5/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= -github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/sjson v1.0.4/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= -github.com/tinylib/msgp v1.1.1/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= -github.com/ugorji/go v1.1.5-pre/go.mod h1:FwP/aQVg39TXzItUBMwnWp9T9gPQnXw4Poh4/oBQZ/0= -github.com/ugorji/go/codec v1.1.5-pre/go.mod h1:tULtS6Gy1AE1yCENaw4Vb//HLH5njI2tfCQDUqRd8fI= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= -github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/willf/bloom v2.0.3+incompatible/go.mod h1:MmAltL9pDMNTrvUkxdg0k0q5I0suxmuwp3KbyrZLOZ8= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= -github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= -golang.org/x/arch v0.0.0-20190909030613-46d78d1859ac/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4= -golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 h1:vEg9joUBmeBcK9iSJftGNf3coIG4HqZElCPehJsfAYM= -golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= -golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200610111108-226ff32320da h1:bGb80FudwxpeucJUjPYJXuJ8Hk91vNtfvrymzwiei38= -golang.org/x/sys v0.0.0-20200610111108-226ff32320da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/pkg/lifecycle/lifecycle.go b/pkg/lifecycle/lifecycle.go new file mode 100644 index 000000000..2812e69c9 --- /dev/null +++ b/pkg/lifecycle/lifecycle.go @@ -0,0 +1,232 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package lifecycle contains all the lifecycle related data types and marshallers. +package lifecycle + +import ( + "encoding/xml" + "time" +) + +// AbortIncompleteMultipartUpload structure, not supported yet on MinIO +type AbortIncompleteMultipartUpload struct { + XMLName xml.Name `xml:"AbortIncompleteMultipartUpload,omitempty" json:"-"` + DaysAfterInitiation int64 `xml:"DaysAfterInitiation,omitempty" json:"DaysAfterInitiation,omitempty"` +} + +// NoncurrentVersionExpiration - Specifies when noncurrent object versions expire. +// Upon expiration, server permanently deletes the noncurrent object versions. +// Set this lifecycle configuration action on a bucket that has versioning enabled +// (or suspended) to request server delete noncurrent object versions at a +// specific period in the object's lifetime. +type NoncurrentVersionExpiration struct { + XMLName xml.Name `xml:"NoncurrentVersionExpiration" json:"-"` + NoncurrentDays ExpirationDays `xml:"NoncurrentDays,omitempty"` +} + +// NoncurrentVersionTransition structure, set this action to request server to +// transition noncurrent object versions to different set storage classes +// at a specific period in the object's lifetime. +type NoncurrentVersionTransition struct { + XMLName xml.Name `xml:"NoncurrentVersionTransition,omitempty" json:"-"` + StorageClass string `xml:"StorageClass,omitempty" json:"StorageClass,omitempty"` + NoncurrentDays ExpirationDays `xml:"NoncurrentDays,omitempty" json:"NoncurrentDays,omitempty"` +} + +// MarshalXML if non-current days not set to non zero value +func (n NoncurrentVersionExpiration) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if n.IsDaysNull() { + return nil + } + type noncurrentVersionExpirationWrapper NoncurrentVersionExpiration + return e.EncodeElement(noncurrentVersionExpirationWrapper(n), start) +} + +// IsDaysNull returns true if days field is null +func (n NoncurrentVersionExpiration) IsDaysNull() bool { + return n.NoncurrentDays == ExpirationDays(0) +} + +// MarshalXML is extended to leave out +// tags +func (n NoncurrentVersionTransition) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if n.NoncurrentDays == ExpirationDays(0) { + return nil + } + return e.EncodeElement(&n, start) +} + +// Tag structure key/value pair representing an object tag to apply lifecycle configuration +type Tag struct { + XMLName xml.Name `xml:"Tag,omitempty" json:"-"` + Key string `xml:"Key,omitempty" json:"Key,omitempty"` + Value string `xml:"Value,omitempty" json:"Value,omitempty"` +} + +// IsEmpty returns whether this tag is empty or not. +func (tag Tag) IsEmpty() bool { + return tag.Key == "" +} + +// Transition structure - transition details of lifecycle configuration +type Transition struct { + XMLName xml.Name `xml:"Transition" json:"-"` + TransitionDate *time.Time `xml:"Date,omitempty" json:"Date,omitempty"` + StorageClass string `xml:"StorageClass,omitempty" json:"StorageClass,omitempty"` + TransitionInDays int `xml:"Days,omitempty" json:"Days,omitempty"` +} + +// IsSet is transition set +func (t Transition) IsSet() bool { + return t.TransitionDate != nil && !t.TransitionDate.IsZero() || t.TransitionInDays > 0 +} + +// And And Rule for LifecycleTag, to be used in LifecycleRuleFilter +type And struct { + XMLName xml.Name `xml:"And,omitempty" json:"-"` + Prefix string `xml:"Prefix,omitempty" json:"Prefix,omitempty"` + Tags []Tag `xml:"Tag,omitempty" json:"Tags,omitempty"` +} + +// IsEmpty returns true if Tags field is null +func (a And) IsEmpty() bool { + return len(a.Tags) == 0 && a.Prefix == "" +} + +// Filter will be used in selecting rule(s) for lifecycle configuration +type Filter struct { + XMLName xml.Name `xml:"Filter" json:"-"` + And And `xml:"And,omitempty" json:"And,omitempty"` + Prefix string `xml:"Prefix,omitempty" json:"Prefix,omitempty"` + Tag Tag `xml:"Tag,omitempty" json:"-"` +} + +// MarshalXML - produces the xml representation of the Filter struct +// only one of Prefix, And and Tag should be present in the output. +func (f Filter) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if err := e.EncodeToken(start); err != nil { + return err + } + + switch { + case !f.And.IsEmpty(): + if err := e.EncodeElement(f.And, xml.StartElement{Name: xml.Name{Local: "And"}}); err != nil { + return err + } + case !f.Tag.IsEmpty(): + if err := e.EncodeElement(f.Tag, xml.StartElement{Name: xml.Name{Local: "Tag"}}); err != nil { + return err + } + default: + // Always print Prefix field when both And & Tag are empty + if err := e.EncodeElement(f.Prefix, xml.StartElement{Name: xml.Name{Local: "Prefix"}}); err != nil { + return err + } + } + + return e.EncodeToken(xml.EndElement{Name: start.Name}) +} + +// ExpirationDays is a type alias to unmarshal Days in Expiration +type ExpirationDays int + +// MarshalXML encodes number of days to expire if it is non-zero and +// encodes empty string otherwise +func (eDays *ExpirationDays) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { + if *eDays == ExpirationDays(0) { + return nil + } + return e.EncodeElement(int(*eDays), startElement) +} + +// ExpirationDate is a embedded type containing time.Time to unmarshal +// Date in Expiration +type ExpirationDate struct { + time.Time +} + +// MarshalXML encodes expiration date if it is non-zero and encodes +// empty string otherwise +func (eDate *ExpirationDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { + if *eDate == (ExpirationDate{time.Time{}}) { + return nil + } + return e.EncodeElement(eDate.Format(time.RFC3339), startElement) +} + +// ExpireDeleteMarker represents value of ExpiredObjectDeleteMarker field in Expiration XML element. +type ExpireDeleteMarker bool + +// MarshalXML encodes delete marker boolean into an XML form. +func (b ExpireDeleteMarker) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { + if !b { + return nil + } + type expireDeleteMarkerWrapper ExpireDeleteMarker + return e.EncodeElement(expireDeleteMarkerWrapper(b), startElement) +} + +// Expiration structure - expiration details of lifecycle configuration +type Expiration struct { + XMLName xml.Name `xml:"Expiration,omitempty" json:"-"` + Date ExpirationDate `xml:"Date,omitempty" json:"Date,omitempty"` + Days ExpirationDays `xml:"Days,omitempty" json:"Days,omitempty"` + DeleteMarker ExpireDeleteMarker `xml:"ExpiredObjectDeleteMarker,omitempty"` +} + +// IsDaysNull returns true if days field is null +func (e Expiration) IsDaysNull() bool { + return e.Days == ExpirationDays(0) +} + +// IsDateNull returns true if date field is null +func (e Expiration) IsDateNull() bool { + return e.Date.Time.IsZero() +} + +// IsNull returns true if both date and days fields are null +func (e Expiration) IsNull() bool { + return e.IsDaysNull() && e.IsDateNull() +} + +// Rule represents a single rule in lifecycle configuration +type Rule struct { + XMLName xml.Name `xml:"Rule,omitempty" json:"-"` + AbortIncompleteMultipartUpload AbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload,omitempty" json:"AbortIncompleteMultipartUpload,omitempty"` + Expiration Expiration `xml:"Expiration,omitempty" json:"Expiration,omitempty"` + ID string `xml:"ID" json:"ID"` + RuleFilter Filter `xml:"Filter,omitempty" json:"Filter,omitempty"` + NoncurrentVersionExpiration NoncurrentVersionExpiration `xml:"NoncurrentVersionExpiration,omitempty" json:"NoncurrentVersionExpiration,omitempty"` + NoncurrentVersionTransition NoncurrentVersionTransition `xml:"NoncurrentVersionTransition,omitempty" json:"NoncurrentVersionTransition,omitempty"` + Prefix string `xml:"Prefix,omitempty" json:"Prefix,omitempty"` + Status string `xml:"Status" json:"Status"` + Transition Transition `xml:"Transition,omitempty" json:"Transition,omitempty"` +} + +// Configuration is a collection of Rule objects. +type Configuration struct { + XMLName xml.Name `xml:"LifecycleConfiguration,omitempty" json:"-"` + Rules []Rule `xml:"Rule"` +} + +// NewConfiguration initializes a fresh lifecycle configuration +// for manipulation, such as setting and removing lifecycle rules +// and filters. +func NewConfiguration() *Configuration { + return &Configuration{} +} diff --git a/pkg/notification/info.go b/pkg/notification/info.go new file mode 100644 index 000000000..d0a471638 --- /dev/null +++ b/pkg/notification/info.go @@ -0,0 +1,78 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017-2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package notification + +// Indentity represents the user id, this is a compliance field. +type identity struct { + PrincipalID string `json:"principalId"` +} + +// event bucket metadata. +type bucketMeta struct { + Name string `json:"name"` + OwnerIdentity identity `json:"ownerIdentity"` + ARN string `json:"arn"` +} + +// event object metadata. +type objectMeta struct { + Key string `json:"key"` + Size int64 `json:"size,omitempty"` + ETag string `json:"eTag,omitempty"` + ContentType string `json:"contentType,omitempty"` + UserMetadata map[string]string `json:"userMetadata,omitempty"` + VersionID string `json:"versionId,omitempty"` + Sequencer string `json:"sequencer"` +} + +// event server specific metadata. +type eventMeta struct { + SchemaVersion string `json:"s3SchemaVersion"` + ConfigurationID string `json:"configurationId"` + Bucket bucketMeta `json:"bucket"` + Object objectMeta `json:"object"` +} + +// sourceInfo represents information on the client that +// triggered the event notification. +type sourceInfo struct { + Host string `json:"host"` + Port string `json:"port"` + UserAgent string `json:"userAgent"` +} + +// Event represents an Amazon an S3 bucket notification event. +type Event struct { + EventVersion string `json:"eventVersion"` + EventSource string `json:"eventSource"` + AwsRegion string `json:"awsRegion"` + EventTime string `json:"eventTime"` + EventName string `json:"eventName"` + UserIdentity identity `json:"userIdentity"` + RequestParameters map[string]string `json:"requestParameters"` + ResponseElements map[string]string `json:"responseElements"` + S3 eventMeta `json:"s3"` + Source sourceInfo `json:"source"` +} + +// Info - represents the collection of notification events, additionally +// also reports errors if any while listening on bucket notifications. +type Info struct { + Records []Event + Err error +} diff --git a/bucket-notification.go b/pkg/notification/notification.go similarity index 68% rename from bucket-notification.go rename to pkg/notification/notification.go index 7a7d578f0..d19bef02b 100644 --- a/bucket-notification.go +++ b/pkg/notification/notification.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2020 MinIO, Inc. + * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ * limitations under the License. */ -package minio +package notification import ( "encoding/xml" @@ -25,24 +25,24 @@ import ( "github.com/minio/minio-go/v7/pkg/set" ) -// NotificationEventType is a S3 notification event associated to the bucket notification configuration -type NotificationEventType string +// EventType is a S3 notification event associated to the bucket notification configuration +type EventType string // The role of all event types are described in : // http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html#notification-how-to-event-types-and-destinations const ( - ObjectCreatedAll NotificationEventType = "s3:ObjectCreated:*" - ObjectCreatedPut = "s3:ObjectCreated:Put" - ObjectCreatedPost = "s3:ObjectCreated:Post" - ObjectCreatedCopy = "s3:ObjectCreated:Copy" - ObjectCreatedCompleteMultipartUpload = "s3:ObjectCreated:CompleteMultipartUpload" - ObjectAccessedGet = "s3:ObjectAccessed:Get" - ObjectAccessedHead = "s3:ObjectAccessed:Head" - ObjectAccessedAll = "s3:ObjectAccessed:*" - ObjectRemovedAll = "s3:ObjectRemoved:*" - ObjectRemovedDelete = "s3:ObjectRemoved:Delete" - ObjectRemovedDeleteMarkerCreated = "s3:ObjectRemoved:DeleteMarkerCreated" - ObjectReducedRedundancyLostObject = "s3:ReducedRedundancyLostObject" + ObjectCreatedAll EventType = "s3:ObjectCreated:*" + ObjectCreatedPut = "s3:ObjectCreated:Put" + ObjectCreatedPost = "s3:ObjectCreated:Post" + ObjectCreatedCopy = "s3:ObjectCreated:Copy" + ObjectCreatedCompleteMultipartUpload = "s3:ObjectCreated:CompleteMultipartUpload" + ObjectAccessedGet = "s3:ObjectAccessed:Get" + ObjectAccessedHead = "s3:ObjectAccessed:Head" + ObjectAccessedAll = "s3:ObjectAccessed:*" + ObjectRemovedAll = "s3:ObjectRemoved:*" + ObjectRemovedDelete = "s3:ObjectRemoved:Delete" + ObjectRemovedDeleteMarkerCreated = "s3:ObjectRemoved:DeleteMarkerCreated" + ObjectReducedRedundancyLostObject = "s3:ReducedRedundancyLostObject" ) // FilterRule - child of S3Key, a tag in the notification xml which @@ -88,27 +88,27 @@ func (arn Arn) String() string { return "arn:" + arn.Partition + ":" + arn.Service + ":" + arn.Region + ":" + arn.AccountID + ":" + arn.Resource } -// NotificationConfig - represents one single notification configuration +// Config - represents one single notification configuration // such as topic, queue or lambda configuration. -type NotificationConfig struct { - ID string `xml:"Id,omitempty"` - Arn Arn `xml:"-"` - Events []NotificationEventType `xml:"Event"` - Filter *Filter `xml:"Filter,omitempty"` +type Config struct { + ID string `xml:"Id,omitempty"` + Arn Arn `xml:"-"` + Events []EventType `xml:"Event"` + Filter *Filter `xml:"Filter,omitempty"` } -// NewNotificationConfig creates one notification config and sets the given ARN -func NewNotificationConfig(arn Arn) NotificationConfig { - return NotificationConfig{Arn: arn, Filter: &Filter{}} +// NewConfig creates one notification config and sets the given ARN +func NewConfig(arn Arn) Config { + return Config{Arn: arn, Filter: &Filter{}} } // AddEvents adds one event to the current notification config -func (t *NotificationConfig) AddEvents(events ...NotificationEventType) { +func (t *Config) AddEvents(events ...EventType) { t.Events = append(t.Events, events...) } // AddFilterSuffix sets the suffix configuration to the current notification config -func (t *NotificationConfig) AddFilterSuffix(suffix string) { +func (t *Config) AddFilterSuffix(suffix string) { if t.Filter == nil { t.Filter = &Filter{} } @@ -124,7 +124,7 @@ func (t *NotificationConfig) AddFilterSuffix(suffix string) { } // AddFilterPrefix sets the prefix configuration to the current notification config -func (t *NotificationConfig) AddFilterPrefix(prefix string) { +func (t *Config) AddFilterPrefix(prefix string) { if t.Filter == nil { t.Filter = &Filter{} } @@ -139,8 +139,8 @@ func (t *NotificationConfig) AddFilterPrefix(prefix string) { t.Filter.S3Key.FilterRules = append(t.Filter.S3Key.FilterRules, newFilterRule) } -// EqualNotificationEventTypeList tells whether a and b contain the same events -func EqualNotificationEventTypeList(a, b []NotificationEventType) bool { +// EqualEventTypeList tells whether a and b contain the same events +func EqualEventTypeList(a, b []EventType) bool { if len(a) != len(b) { return false } @@ -176,10 +176,10 @@ func EqualFilterRuleList(a, b []FilterRule) bool { return setA.Difference(setB).IsEmpty() } -// Equal returns whether this `NotificationConfig` is equal to another defined by the passed parameters -func (t *NotificationConfig) Equal(events []NotificationEventType, prefix, suffix string) bool { +// Equal returns whether this `Config` is equal to another defined by the passed parameters +func (t *Config) Equal(events []EventType, prefix, suffix string) bool { //Compare events - passEvents := EqualNotificationEventTypeList(t.Events, events) + passEvents := EqualEventTypeList(t.Events, events) //Compare filters var newFilter []FilterRule @@ -197,33 +197,33 @@ func (t *NotificationConfig) Equal(events []NotificationEventType, prefix, suffi // TopicConfig carries one single topic notification configuration type TopicConfig struct { - NotificationConfig + Config Topic string `xml:"Topic"` } // QueueConfig carries one single queue notification configuration type QueueConfig struct { - NotificationConfig + Config Queue string `xml:"Queue"` } // LambdaConfig carries one single cloudfunction notification configuration type LambdaConfig struct { - NotificationConfig + Config Lambda string `xml:"CloudFunction"` } -// BucketNotification - the struct that represents the whole XML to be sent to the web service -type BucketNotification struct { - XMLName xml.Name `xml:"NotificationConfiguration"` +// Configuration - the struct that represents the whole XML to be sent to the web service +type Configuration struct { + XMLName xml.Name `xml:"Configuration"` LambdaConfigs []LambdaConfig `xml:"CloudFunctionConfiguration"` TopicConfigs []TopicConfig `xml:"TopicConfiguration"` QueueConfigs []QueueConfig `xml:"QueueConfiguration"` } // AddTopic adds a given topic config to the general bucket notification config -func (b *BucketNotification) AddTopic(topicConfig NotificationConfig) bool { - newTopicConfig := TopicConfig{NotificationConfig: topicConfig, Topic: topicConfig.Arn.String()} +func (b *Configuration) AddTopic(topicConfig Config) bool { + newTopicConfig := TopicConfig{Config: topicConfig, Topic: topicConfig.Arn.String()} for _, n := range b.TopicConfigs { // If new config matches existing one if n.Topic == newTopicConfig.Arn.String() && newTopicConfig.Filter == n.Filter { @@ -248,8 +248,8 @@ func (b *BucketNotification) AddTopic(topicConfig NotificationConfig) bool { } // AddQueue adds a given queue config to the general bucket notification config -func (b *BucketNotification) AddQueue(queueConfig NotificationConfig) bool { - newQueueConfig := QueueConfig{NotificationConfig: queueConfig, Queue: queueConfig.Arn.String()} +func (b *Configuration) AddQueue(queueConfig Config) bool { + newQueueConfig := QueueConfig{Config: queueConfig, Queue: queueConfig.Arn.String()} for _, n := range b.QueueConfigs { if n.Queue == newQueueConfig.Arn.String() && newQueueConfig.Filter == n.Filter { @@ -273,8 +273,8 @@ func (b *BucketNotification) AddQueue(queueConfig NotificationConfig) bool { } // AddLambda adds a given lambda config to the general bucket notification config -func (b *BucketNotification) AddLambda(lambdaConfig NotificationConfig) bool { - newLambdaConfig := LambdaConfig{NotificationConfig: lambdaConfig, Lambda: lambdaConfig.Arn.String()} +func (b *Configuration) AddLambda(lambdaConfig Config) bool { + newLambdaConfig := LambdaConfig{Config: lambdaConfig, Lambda: lambdaConfig.Arn.String()} for _, n := range b.LambdaConfigs { if n.Lambda == newLambdaConfig.Arn.String() && newLambdaConfig.Filter == n.Filter { @@ -298,7 +298,7 @@ func (b *BucketNotification) AddLambda(lambdaConfig NotificationConfig) bool { } // RemoveTopicByArn removes all topic configurations that match the exact specified ARN -func (b *BucketNotification) RemoveTopicByArn(arn Arn) { +func (b *Configuration) RemoveTopicByArn(arn Arn) { var topics []TopicConfig for _, topic := range b.TopicConfigs { if topic.Topic != arn.String() { @@ -308,15 +308,15 @@ func (b *BucketNotification) RemoveTopicByArn(arn Arn) { b.TopicConfigs = topics } -// ErrNoNotificationConfigMatch is returned when a notification configuration (sqs,sns,lambda) is not found when trying to delete -var ErrNoNotificationConfigMatch = errors.New("no notification configuration matched") +// ErrNoConfigMatch is returned when a notification configuration (sqs,sns,lambda) is not found when trying to delete +var ErrNoConfigMatch = errors.New("no notification configuration matched") // RemoveTopicByArnEventsPrefixSuffix removes a topic configuration that match the exact specified ARN, events, prefix and suffix -func (b *BucketNotification) RemoveTopicByArnEventsPrefixSuffix(arn Arn, events []NotificationEventType, prefix, suffix string) error { +func (b *Configuration) RemoveTopicByArnEventsPrefixSuffix(arn Arn, events []EventType, prefix, suffix string) error { removeIndex := -1 for i, v := range b.TopicConfigs { // if it matches events and filters, mark the index for deletion - if v.Topic == arn.String() && v.NotificationConfig.Equal(events, prefix, suffix) { + if v.Topic == arn.String() && v.Config.Equal(events, prefix, suffix) { removeIndex = i break // since we have at most one matching config } @@ -325,11 +325,11 @@ func (b *BucketNotification) RemoveTopicByArnEventsPrefixSuffix(arn Arn, events b.TopicConfigs = append(b.TopicConfigs[:removeIndex], b.TopicConfigs[removeIndex+1:]...) return nil } - return ErrNoNotificationConfigMatch + return ErrNoConfigMatch } // RemoveQueueByArn removes all queue configurations that match the exact specified ARN -func (b *BucketNotification) RemoveQueueByArn(arn Arn) { +func (b *Configuration) RemoveQueueByArn(arn Arn) { var queues []QueueConfig for _, queue := range b.QueueConfigs { if queue.Queue != arn.String() { @@ -340,11 +340,11 @@ func (b *BucketNotification) RemoveQueueByArn(arn Arn) { } // RemoveQueueByArnEventsPrefixSuffix removes a queue configuration that match the exact specified ARN, events, prefix and suffix -func (b *BucketNotification) RemoveQueueByArnEventsPrefixSuffix(arn Arn, events []NotificationEventType, prefix, suffix string) error { +func (b *Configuration) RemoveQueueByArnEventsPrefixSuffix(arn Arn, events []EventType, prefix, suffix string) error { removeIndex := -1 for i, v := range b.QueueConfigs { // if it matches events and filters, mark the index for deletion - if v.Queue == arn.String() && v.NotificationConfig.Equal(events, prefix, suffix) { + if v.Queue == arn.String() && v.Config.Equal(events, prefix, suffix) { removeIndex = i break // since we have at most one matching config } @@ -353,11 +353,11 @@ func (b *BucketNotification) RemoveQueueByArnEventsPrefixSuffix(arn Arn, events b.QueueConfigs = append(b.QueueConfigs[:removeIndex], b.QueueConfigs[removeIndex+1:]...) return nil } - return ErrNoNotificationConfigMatch + return ErrNoConfigMatch } // RemoveLambdaByArn removes all lambda configurations that match the exact specified ARN -func (b *BucketNotification) RemoveLambdaByArn(arn Arn) { +func (b *Configuration) RemoveLambdaByArn(arn Arn) { var lambdas []LambdaConfig for _, lambda := range b.LambdaConfigs { if lambda.Lambda != arn.String() { @@ -368,11 +368,11 @@ func (b *BucketNotification) RemoveLambdaByArn(arn Arn) { } // RemoveLambdaByArnEventsPrefixSuffix removes a topic configuration that match the exact specified ARN, events, prefix and suffix -func (b *BucketNotification) RemoveLambdaByArnEventsPrefixSuffix(arn Arn, events []NotificationEventType, prefix, suffix string) error { +func (b *Configuration) RemoveLambdaByArnEventsPrefixSuffix(arn Arn, events []EventType, prefix, suffix string) error { removeIndex := -1 for i, v := range b.LambdaConfigs { // if it matches events and filters, mark the index for deletion - if v.Lambda == arn.String() && v.NotificationConfig.Equal(events, prefix, suffix) { + if v.Lambda == arn.String() && v.Config.Equal(events, prefix, suffix) { removeIndex = i break // since we have at most one matching config } @@ -381,5 +381,5 @@ func (b *BucketNotification) RemoveLambdaByArnEventsPrefixSuffix(arn Arn, events b.LambdaConfigs = append(b.LambdaConfigs[:removeIndex], b.LambdaConfigs[removeIndex+1:]...) return nil } - return ErrNoNotificationConfigMatch + return ErrNoConfigMatch } diff --git a/bucket-notification_test.go b/pkg/notification/notification_test.go similarity index 83% rename from bucket-notification_test.go rename to pkg/notification/notification_test.go index 74d43352e..ac0db754d 100644 --- a/bucket-notification_test.go +++ b/pkg/notification/notification_test.go @@ -15,17 +15,17 @@ * limitations under the License. */ -package minio +package notification import ( "encoding/xml" "testing" ) -func TestEqualNotificationEventTypeList(t *testing.T) { +func TestEqualEventTypeList(t *testing.T) { type args struct { - a []NotificationEventType - b []NotificationEventType + a []EventType + b []EventType } tests := []struct { name string @@ -35,31 +35,31 @@ func TestEqualNotificationEventTypeList(t *testing.T) { { name: "same order", args: args{ - a: []NotificationEventType{ObjectCreatedAll, ObjectAccessedAll}, - b: []NotificationEventType{ObjectCreatedAll, ObjectAccessedAll}, + a: []EventType{ObjectCreatedAll, ObjectAccessedAll}, + b: []EventType{ObjectCreatedAll, ObjectAccessedAll}, }, want: true, }, { name: "different order", args: args{ - a: []NotificationEventType{ObjectCreatedAll, ObjectAccessedAll}, - b: []NotificationEventType{ObjectAccessedAll, ObjectCreatedAll}, + a: []EventType{ObjectCreatedAll, ObjectAccessedAll}, + b: []EventType{ObjectAccessedAll, ObjectCreatedAll}, }, want: true, }, { name: "not equal", args: args{ - a: []NotificationEventType{ObjectCreatedAll, ObjectAccessedAll}, - b: []NotificationEventType{ObjectRemovedAll, ObjectAccessedAll}, + a: []EventType{ObjectCreatedAll, ObjectAccessedAll}, + b: []EventType{ObjectRemovedAll, ObjectAccessedAll}, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := EqualNotificationEventTypeList(tt.args.a, tt.args.b); got != tt.want { - t.Errorf("EqualNotificationEventTypeList() = %v, want %v", got, tt.want) + if got := EqualEventTypeList(tt.args.a, tt.args.b); got != tt.want { + t.Errorf("EqualEventTypeList() = %v, want %v", got, tt.want) } }) } @@ -108,15 +108,15 @@ func TestEqualFilterRuleList(t *testing.T) { } } -func TestNotificationConfig_Equal(t *testing.T) { +func TestConfig_Equal(t *testing.T) { type fields struct { ID string Arn Arn - Events []NotificationEventType + Events []EventType Filter *Filter } type args struct { - events []NotificationEventType + events []EventType prefix string suffix string } @@ -130,7 +130,7 @@ func TestNotificationConfig_Equal(t *testing.T) { name: "same order", fields: fields{ Arn: NewArn("minio", "sqs", "", "1", "postgresql"), - Events: []NotificationEventType{ObjectCreatedAll, ObjectAccessedAll}, + Events: []EventType{ObjectCreatedAll, ObjectAccessedAll}, Filter: &Filter{ S3Key: S3Key{ FilterRules: []FilterRule{{Name: "prefix", Value: "prefix1"}, {Name: "suffix", Value: "suffix1"}}, @@ -138,7 +138,7 @@ func TestNotificationConfig_Equal(t *testing.T) { }, }, args: args{ - events: []NotificationEventType{ObjectCreatedAll, ObjectAccessedAll}, + events: []EventType{ObjectCreatedAll, ObjectAccessedAll}, prefix: "prefix1", suffix: "suffix1", }, @@ -148,7 +148,7 @@ func TestNotificationConfig_Equal(t *testing.T) { name: "different order", fields: fields{ Arn: NewArn("minio", "sqs", "", "1", "postgresql"), - Events: []NotificationEventType{ObjectAccessedAll, ObjectCreatedAll}, + Events: []EventType{ObjectAccessedAll, ObjectCreatedAll}, Filter: &Filter{ S3Key: S3Key{ FilterRules: []FilterRule{{Name: "suffix", Value: "suffix1"}, {Name: "prefix", Value: "prefix1"}}, @@ -156,7 +156,7 @@ func TestNotificationConfig_Equal(t *testing.T) { }, }, args: args{ - events: []NotificationEventType{ObjectCreatedAll, ObjectAccessedAll}, + events: []EventType{ObjectCreatedAll, ObjectAccessedAll}, prefix: "prefix1", suffix: "suffix1", }, @@ -166,7 +166,7 @@ func TestNotificationConfig_Equal(t *testing.T) { name: "not equal", fields: fields{ Arn: NewArn("minio", "sqs", "", "1", "postgresql"), - Events: []NotificationEventType{ObjectAccessedAll}, + Events: []EventType{ObjectAccessedAll}, Filter: &Filter{ S3Key: S3Key{ FilterRules: []FilterRule{{Name: "suffix", Value: "suffix1"}, {Name: "prefix", Value: "prefix1"}}, @@ -174,7 +174,7 @@ func TestNotificationConfig_Equal(t *testing.T) { }, }, args: args{ - events: []NotificationEventType{ObjectCreatedAll, ObjectAccessedAll}, + events: []EventType{ObjectCreatedAll, ObjectAccessedAll}, prefix: "prefix1", suffix: "suffix1", }, @@ -183,7 +183,7 @@ func TestNotificationConfig_Equal(t *testing.T) { { name: "different arn", fields: fields{ - Events: []NotificationEventType{ObjectAccessedAll}, + Events: []EventType{ObjectAccessedAll}, Filter: &Filter{ S3Key: S3Key{ FilterRules: []FilterRule{{Name: "suffix", Value: "suffix1"}, {Name: "prefix", Value: "prefix1"}}, @@ -191,7 +191,7 @@ func TestNotificationConfig_Equal(t *testing.T) { }, }, args: args{ - events: []NotificationEventType{ObjectCreatedAll, ObjectAccessedAll}, + events: []EventType{ObjectCreatedAll, ObjectAccessedAll}, prefix: "prefix1", suffix: "suffix1", }, @@ -200,7 +200,7 @@ func TestNotificationConfig_Equal(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - nc := &NotificationConfig{ + nc := &Config{ ID: tt.fields.ID, Arn: tt.fields.Arn, Events: tt.fields.Events, @@ -213,7 +213,7 @@ func TestNotificationConfig_Equal(t *testing.T) { } } -func TestBucketNotification_RemoveQueueByArnEventsPrefixSuffix(t *testing.T) { +func TestConfiguration_RemoveQueueByArnEventsPrefixSuffix(t *testing.T) { type fields struct { XMLName xml.Name LambdaConfigs []LambdaConfig @@ -222,7 +222,7 @@ func TestBucketNotification_RemoveQueueByArnEventsPrefixSuffix(t *testing.T) { } type args struct { arn Arn - events []NotificationEventType + events []EventType prefix string suffix string } @@ -240,7 +240,7 @@ func TestBucketNotification_RemoveQueueByArnEventsPrefixSuffix(t *testing.T) { TopicConfigs: nil, QueueConfigs: []QueueConfig{ { - NotificationConfig: NotificationConfig{ + Config: Config{ ID: "", Arn: Arn{ Partition: "minio", @@ -249,7 +249,7 @@ func TestBucketNotification_RemoveQueueByArnEventsPrefixSuffix(t *testing.T) { AccountID: "1", Resource: "postgresql", }, - Events: []NotificationEventType{ + Events: []EventType{ ObjectAccessedAll, }, Filter: &Filter{ @@ -275,7 +275,7 @@ func TestBucketNotification_RemoveQueueByArnEventsPrefixSuffix(t *testing.T) { AccountID: "1", Resource: "postgresql", }, - events: []NotificationEventType{ + events: []EventType{ ObjectAccessedAll, }, prefix: "x", @@ -291,7 +291,7 @@ func TestBucketNotification_RemoveQueueByArnEventsPrefixSuffix(t *testing.T) { TopicConfigs: nil, QueueConfigs: []QueueConfig{ { - NotificationConfig: NotificationConfig{ + Config: Config{ ID: "", Arn: Arn{ Partition: "minio", @@ -300,7 +300,7 @@ func TestBucketNotification_RemoveQueueByArnEventsPrefixSuffix(t *testing.T) { AccountID: "1", Resource: "postgresql", }, - Events: []NotificationEventType{ + Events: []EventType{ ObjectAccessedAll, }, Filter: &Filter{ @@ -330,7 +330,7 @@ func TestBucketNotification_RemoveQueueByArnEventsPrefixSuffix(t *testing.T) { AccountID: "1", Resource: "postgresql", }, - events: []NotificationEventType{ + events: []EventType{ ObjectAccessedAll, }, prefix: "x", @@ -346,7 +346,7 @@ func TestBucketNotification_RemoveQueueByArnEventsPrefixSuffix(t *testing.T) { TopicConfigs: nil, QueueConfigs: []QueueConfig{ { - NotificationConfig: NotificationConfig{ + Config: Config{ ID: "", Arn: Arn{ Partition: "minio", @@ -355,7 +355,7 @@ func TestBucketNotification_RemoveQueueByArnEventsPrefixSuffix(t *testing.T) { AccountID: "1", Resource: "postgresql", }, - Events: []NotificationEventType{ + Events: []EventType{ ObjectAccessedAll, }, Filter: &Filter{ @@ -381,7 +381,7 @@ func TestBucketNotification_RemoveQueueByArnEventsPrefixSuffix(t *testing.T) { AccountID: "1", Resource: "postgresql", }, - events: []NotificationEventType{ + events: []EventType{ ObjectAccessedAll, }, prefix: "", @@ -392,7 +392,7 @@ func TestBucketNotification_RemoveQueueByArnEventsPrefixSuffix(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - b := &BucketNotification{ + b := &Configuration{ XMLName: tt.fields.XMLName, LambdaConfigs: tt.fields.LambdaConfigs, TopicConfigs: tt.fields.TopicConfigs, @@ -405,7 +405,7 @@ func TestBucketNotification_RemoveQueueByArnEventsPrefixSuffix(t *testing.T) { } } -func TestBucketNotification_RemoveLambdaByArnEventsPrefixSuffix(t *testing.T) { +func TestConfiguration_RemoveLambdaByArnEventsPrefixSuffix(t *testing.T) { type fields struct { XMLName xml.Name LambdaConfigs []LambdaConfig @@ -414,7 +414,7 @@ func TestBucketNotification_RemoveLambdaByArnEventsPrefixSuffix(t *testing.T) { } type args struct { arn Arn - events []NotificationEventType + events []EventType prefix string suffix string } @@ -432,7 +432,7 @@ func TestBucketNotification_RemoveLambdaByArnEventsPrefixSuffix(t *testing.T) { TopicConfigs: nil, LambdaConfigs: []LambdaConfig{ { - NotificationConfig: NotificationConfig{ + Config: Config{ ID: "", Arn: Arn{ Partition: "minio", @@ -441,7 +441,7 @@ func TestBucketNotification_RemoveLambdaByArnEventsPrefixSuffix(t *testing.T) { AccountID: "1", Resource: "provider", }, - Events: []NotificationEventType{ + Events: []EventType{ ObjectAccessedAll, }, Filter: &Filter{ @@ -467,7 +467,7 @@ func TestBucketNotification_RemoveLambdaByArnEventsPrefixSuffix(t *testing.T) { AccountID: "1", Resource: "provider", }, - events: []NotificationEventType{ + events: []EventType{ ObjectAccessedAll, }, prefix: "x", @@ -483,7 +483,7 @@ func TestBucketNotification_RemoveLambdaByArnEventsPrefixSuffix(t *testing.T) { TopicConfigs: nil, LambdaConfigs: []LambdaConfig{ { - NotificationConfig: NotificationConfig{ + Config: Config{ ID: "", Arn: Arn{ Partition: "minio", @@ -492,7 +492,7 @@ func TestBucketNotification_RemoveLambdaByArnEventsPrefixSuffix(t *testing.T) { AccountID: "1", Resource: "provider", }, - Events: []NotificationEventType{ + Events: []EventType{ ObjectAccessedAll, }, Filter: &Filter{ @@ -522,7 +522,7 @@ func TestBucketNotification_RemoveLambdaByArnEventsPrefixSuffix(t *testing.T) { AccountID: "1", Resource: "provider", }, - events: []NotificationEventType{ + events: []EventType{ ObjectAccessedAll, }, prefix: "x", @@ -538,7 +538,7 @@ func TestBucketNotification_RemoveLambdaByArnEventsPrefixSuffix(t *testing.T) { TopicConfigs: nil, LambdaConfigs: []LambdaConfig{ { - NotificationConfig: NotificationConfig{ + Config: Config{ ID: "", Arn: Arn{ Partition: "minio", @@ -547,7 +547,7 @@ func TestBucketNotification_RemoveLambdaByArnEventsPrefixSuffix(t *testing.T) { AccountID: "1", Resource: "provider", }, - Events: []NotificationEventType{ + Events: []EventType{ ObjectAccessedAll, }, Filter: &Filter{ @@ -573,7 +573,7 @@ func TestBucketNotification_RemoveLambdaByArnEventsPrefixSuffix(t *testing.T) { AccountID: "1", Resource: "provider", }, - events: []NotificationEventType{ + events: []EventType{ ObjectAccessedAll, }, prefix: "", @@ -589,7 +589,7 @@ func TestBucketNotification_RemoveLambdaByArnEventsPrefixSuffix(t *testing.T) { TopicConfigs: nil, LambdaConfigs: []LambdaConfig{ { - NotificationConfig: NotificationConfig{ + Config: Config{ ID: "", Arn: Arn{ Partition: "minio", @@ -598,7 +598,7 @@ func TestBucketNotification_RemoveLambdaByArnEventsPrefixSuffix(t *testing.T) { AccountID: "1", Resource: "provider", }, - Events: []NotificationEventType{ + Events: []EventType{ ObjectAccessedAll, }, Filter: &Filter{ @@ -625,7 +625,7 @@ func TestBucketNotification_RemoveLambdaByArnEventsPrefixSuffix(t *testing.T) { AccountID: "2", Resource: "provider", }, - events: []NotificationEventType{ + events: []EventType{ ObjectAccessedAll, }, prefix: "", @@ -636,7 +636,7 @@ func TestBucketNotification_RemoveLambdaByArnEventsPrefixSuffix(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - b := &BucketNotification{ + b := &Configuration{ XMLName: tt.fields.XMLName, LambdaConfigs: tt.fields.LambdaConfigs, TopicConfigs: tt.fields.TopicConfigs, @@ -649,7 +649,7 @@ func TestBucketNotification_RemoveLambdaByArnEventsPrefixSuffix(t *testing.T) { } } -func TestBucketNotification_RemoveTopicByArnEventsPrefixSuffix(t *testing.T) { +func TestConfiguration_RemoveTopicByArnEventsPrefixSuffix(t *testing.T) { type fields struct { XMLName xml.Name LambdaConfigs []LambdaConfig @@ -658,7 +658,7 @@ func TestBucketNotification_RemoveTopicByArnEventsPrefixSuffix(t *testing.T) { } type args struct { arn Arn - events []NotificationEventType + events []EventType prefix string suffix string } @@ -676,7 +676,7 @@ func TestBucketNotification_RemoveTopicByArnEventsPrefixSuffix(t *testing.T) { LambdaConfigs: nil, TopicConfigs: []TopicConfig{ { - NotificationConfig: NotificationConfig{ + Config: Config{ ID: "", Arn: Arn{ Partition: "minio", @@ -685,7 +685,7 @@ func TestBucketNotification_RemoveTopicByArnEventsPrefixSuffix(t *testing.T) { AccountID: "1", Resource: "kafka", }, - Events: []NotificationEventType{ + Events: []EventType{ ObjectAccessedAll, }, Filter: &Filter{ @@ -711,7 +711,7 @@ func TestBucketNotification_RemoveTopicByArnEventsPrefixSuffix(t *testing.T) { AccountID: "1", Resource: "kafka", }, - events: []NotificationEventType{ + events: []EventType{ ObjectAccessedAll, }, prefix: "x", @@ -727,7 +727,7 @@ func TestBucketNotification_RemoveTopicByArnEventsPrefixSuffix(t *testing.T) { LambdaConfigs: nil, TopicConfigs: []TopicConfig{ { - NotificationConfig: NotificationConfig{ + Config: Config{ ID: "", Arn: Arn{ Partition: "minio", @@ -736,7 +736,7 @@ func TestBucketNotification_RemoveTopicByArnEventsPrefixSuffix(t *testing.T) { AccountID: "1", Resource: "kafka", }, - Events: []NotificationEventType{ + Events: []EventType{ ObjectAccessedAll, }, Filter: &Filter{ @@ -766,7 +766,7 @@ func TestBucketNotification_RemoveTopicByArnEventsPrefixSuffix(t *testing.T) { AccountID: "1", Resource: "kafka", }, - events: []NotificationEventType{ + events: []EventType{ ObjectAccessedAll, }, prefix: "x", @@ -782,7 +782,7 @@ func TestBucketNotification_RemoveTopicByArnEventsPrefixSuffix(t *testing.T) { LambdaConfigs: nil, TopicConfigs: []TopicConfig{ { - NotificationConfig: NotificationConfig{ + Config: Config{ ID: "", Arn: Arn{ Partition: "minio", @@ -791,7 +791,7 @@ func TestBucketNotification_RemoveTopicByArnEventsPrefixSuffix(t *testing.T) { AccountID: "1", Resource: "kafka", }, - Events: []NotificationEventType{ + Events: []EventType{ ObjectAccessedAll, }, Filter: &Filter{ @@ -817,7 +817,7 @@ func TestBucketNotification_RemoveTopicByArnEventsPrefixSuffix(t *testing.T) { AccountID: "1", Resource: "kafka", }, - events: []NotificationEventType{ + events: []EventType{ ObjectAccessedAll, }, prefix: "", @@ -828,7 +828,7 @@ func TestBucketNotification_RemoveTopicByArnEventsPrefixSuffix(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - b := &BucketNotification{ + b := &Configuration{ XMLName: tt.fields.XMLName, LambdaConfigs: tt.fields.LambdaConfigs, TopicConfigs: tt.fields.TopicConfigs, diff --git a/pkg/replication/replication.go b/pkg/replication/replication.go index f4818e992..f21720269 100644 --- a/pkg/replication/replication.go +++ b/pkg/replication/replication.go @@ -23,8 +23,8 @@ import "encoding/xml" type Config struct { XMLName xml.Name `xml:"ReplicationConfiguration" json:"-"` Rules []Rule `xml:"Rule" json:"Rules"` - // ReplicationArn is a MinIO only extension and optional for AWS - ReplicationArn string `xml:"ReplicationArn,omitempty" json:"ReplicationArn,omitempty"` + // ReplicationARN is a MinIO only extension and optional for AWS + ReplicationARN string `xml:"ReplicationArn,omitempty" json:"ReplicationArn,omitempty"` } // Empty returns true if config is not set diff --git a/pkg/sse/sse.go b/pkg/sse/sse.go new file mode 100644 index 000000000..b5fb9565a --- /dev/null +++ b/pkg/sse/sse.go @@ -0,0 +1,66 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sse + +import "encoding/xml" + +// ApplySSEByDefault defines default encryption configuration, KMS or SSE. To activate +// KMS, SSEAlgoritm needs to be set to "aws:kms" +// Minio currently does not support Kms. +type ApplySSEByDefault struct { + KmsMasterKeyID string `xml:"KMSMasterKeyID,omitempty"` + SSEAlgorithm string `xml:"SSEAlgorithm"` +} + +// Rule layer encapsulates default encryption configuration +type Rule struct { + Apply ApplySSEByDefault `xml:"ApplyServerSideEncryptionByDefault"` +} + +// Configuration is the default encryption configuration structure +type Configuration struct { + XMLName xml.Name `xml:"ServerSideEncryptionConfiguration"` + Rules []Rule `xml:"Rule"` +} + +// NewConfigurationSSES3 initializes a new SSE-S3 configuration +func NewConfigurationSSES3() *Configuration { + return &Configuration{ + Rules: []Rule{ + { + Apply: ApplySSEByDefault{ + SSEAlgorithm: "AES256", + }, + }, + }, + } +} + +// NewConfigurationSSEKMS initializes a new SSE-KMS configuration +func NewConfigurationSSEKMS(kmsMasterKey string) *Configuration { + return &Configuration{ + Rules: []Rule{ + { + Apply: ApplySSEByDefault{ + KmsMasterKeyID: kmsMasterKey, + SSEAlgorithm: "aws:kms", + }, + }, + }, + } +} From 3818f4cbab631e9d42f7d7753fc986f76a8c574a Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 16 Jul 2020 12:20:27 -0700 Subject: [PATCH 186/215] marshal dest encryption params when SSE-C (#1340) --- api-compose-object.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api-compose-object.go b/api-compose-object.go index a949da861..421850a1d 100644 --- a/api-compose-object.go +++ b/api-compose-object.go @@ -454,6 +454,9 @@ func (c Client) ComposeObject(ctx context.Context, dst CopyDestOptions, srcs ... for i, src := range srcs { var h = make(http.Header) src.Marshal(h) + if dst.Encryption != nil { + dst.Encryption.Marshal(h) + } // calculate start/end indices of parts after // splitting. From 8fd7465ffa8cf2bcb7dce71a9294bfd935862057 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 16 Jul 2020 15:44:31 -0700 Subject: [PATCH 187/215] fix: XML name in notification configuration --- pkg/notification/notification.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/notification/notification.go b/pkg/notification/notification.go index d19bef02b..55c58cb3b 100644 --- a/pkg/notification/notification.go +++ b/pkg/notification/notification.go @@ -215,7 +215,7 @@ type LambdaConfig struct { // Configuration - the struct that represents the whole XML to be sent to the web service type Configuration struct { - XMLName xml.Name `xml:"Configuration"` + XMLName xml.Name `xml:"NotificationConfiguration"` LambdaConfigs []LambdaConfig `xml:"CloudFunctionConfiguration"` TopicConfigs []TopicConfig `xml:"TopicConfiguration"` QueueConfigs []QueueConfig `xml:"QueueConfiguration"` From d1d9114e0ea4a244c629d397926dbef32a2c8273 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 16 Jul 2020 16:59:47 -0700 Subject: [PATCH 188/215] fix: objectTagging APIs to follow bucketTagging (#1341) --- api-object-tagging.go | 18 ++++-------------- docs/API.md | 10 +++++----- examples/s3/getbuckettagging.go | 1 + examples/s3/getobjecttagging.go | 2 +- examples/s3/putobjecttagging.go | 9 +++++++-- examples/s3/setbuckettagging.go | 6 +++--- functional_tests.go | 23 ++++++++++++++++++----- pkg/lifecycle/lifecycle.go | 18 ++++++++++++++++-- pkg/tags/tags.go | 12 ++++++++++++ 9 files changed, 67 insertions(+), 32 deletions(-) diff --git a/api-object-tagging.go b/api-object-tagging.go index 4152366b4..ad0775870 100644 --- a/api-object-tagging.go +++ b/api-object-tagging.go @@ -36,7 +36,7 @@ type PutObjectTaggingOptions struct { // PutObjectTagging replaces or creates object tag(s) and can target // a specific object version in a versioned bucket. -func (c Client) PutObjectTagging(ctx context.Context, bucketName, objectName string, objectTags map[string]string, opts PutObjectTaggingOptions) error { +func (c Client) PutObjectTagging(ctx context.Context, bucketName, objectName string, otags *tags.Tags, opts PutObjectTaggingOptions) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -51,12 +51,7 @@ func (c Client) PutObjectTagging(ctx context.Context, bucketName, objectName str urlValues.Set("versionId", opts.VersionID) } - tags, err := tags.NewTags(objectTags, true) - if err != nil { - return err - } - - reqBytes, err := xml.Marshal(tags) + reqBytes, err := xml.Marshal(otags) if err != nil { return err } @@ -92,7 +87,7 @@ type GetObjectTaggingOptions struct { // GetObjectTagging fetches object tag(s) with options to target // a specific object version in a versioned bucket. -func (c Client) GetObjectTagging(ctx context.Context, bucketName, objectName string, opts GetObjectTaggingOptions) (map[string]string, error) { +func (c Client) GetObjectTagging(ctx context.Context, bucketName, objectName string, opts GetObjectTaggingOptions) (*tags.Tags, error) { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) @@ -120,12 +115,7 @@ func (c Client) GetObjectTagging(ctx context.Context, bucketName, objectName str } } - tags, err := tags.ParseObjectXML(resp.Body) - if err != nil { - return nil, err - } - - return tags.ToMap(), err + return tags.ParseObjectXML(resp.Body) } // RemoveObjectTaggingOptions holds the version id of the object to remove diff --git a/docs/API.md b/docs/API.md index 6c7839b4a..5fdadb24d 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1046,8 +1046,8 @@ __Return Values__ ``` -### PutObjectTagging(ctx context.Context, bucketName, objectName string, objectTags map[string]string) error -Adds or replace Object Tags to the given object +### PutObjectTagging(ctx context.Context, bucketName, objectName string, otags *tags.Tags) error +set new object Tags to the given object, replaces/overwrites any existing tags. __Parameters__ @@ -1057,7 +1057,7 @@ __Parameters__ |`ctx` | _context.Context_ | Custom context for timeout/cancellation of the call| |`bucketName` | _string_ |Name of the bucket | |`objectName` | _string_ |Name of the object | -|`objectTags` | _map[string]string_ | Map with Object Tag's Key and Value | +|`objectTags` | _*tags.Tags_ | Map with Object Tag's Key and Value | __Example__ @@ -1071,7 +1071,7 @@ if err != nil { ``` -### GetObjectTagging(ctx context.Context, bucketName, objectName string) (_map[string]string_, error) +### GetObjectTagging(ctx context.Context, bucketName, objectName string) (*tags.Tags, error) Fetch Object Tags from the given object __Parameters__ @@ -1092,7 +1092,7 @@ if err != nil { fmt.Println(err) return } -fmt.Printf("Fetched Tags: %v", tags) +fmt.Printf("Fetched Tags: %s", tags) ``` diff --git a/examples/s3/getbuckettagging.go b/examples/s3/getbuckettagging.go index 2e7939849..db8dd2040 100644 --- a/examples/s3/getbuckettagging.go +++ b/examples/s3/getbuckettagging.go @@ -40,6 +40,7 @@ func main() { if err != nil { log.Fatalln(err) } + tags, err := s3Client.GetBucketTagging(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) diff --git a/examples/s3/getobjecttagging.go b/examples/s3/getobjecttagging.go index 9d03a5305..4c189377e 100644 --- a/examples/s3/getobjecttagging.go +++ b/examples/s3/getobjecttagging.go @@ -46,5 +46,5 @@ func main() { log.Fatalln(err) } - fmt.Printf("Fetched Object Tags: %v", tags) + fmt.Printf("Fetched Object Tags: %s", tags) } diff --git a/examples/s3/putobjecttagging.go b/examples/s3/putobjecttagging.go index 32fcbd248..722ae875c 100644 --- a/examples/s3/putobjecttagging.go +++ b/examples/s3/putobjecttagging.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/tags" ) func main() { @@ -39,11 +40,15 @@ func main() { if err != nil { log.Fatalln(err) } - tags := map[string]string{ + tagMap := map[string]string{ "Tag1": "Value1", "Tag2": "Value2", } - err = s3Client.PutObjectTagging(context.Background(), "my-bucketname", "my-objectname", tags) + t, err := tags.MapToObjectTags(tagMap) + if err != nil { + log.Fatalln(err) + } + err = s3Client.PutObjectTagging(context.Background(), "my-bucketname", "my-objectname", t) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/setbuckettagging.go b/examples/s3/setbuckettagging.go index 45efc28b2..471f515cd 100644 --- a/examples/s3/setbuckettagging.go +++ b/examples/s3/setbuckettagging.go @@ -41,15 +41,15 @@ func main() { log.Fatalln(err) } - tags, err := tags.NewTags(map[string]string{ + t, err := tags.MapToBucketTags(map[string]string{ "Tag1": "Value1", "Tag2": "Value2", - }, false) + }) if err != nil { log.Fatalln(err) } - err = s3Client.SetBucketTagging(context.Background(), "my-bucketname", tags) + err = s3Client.SetBucketTagging(context.Background(), "my-bucketname", t) if err != nil { log.Fatalln(err) } diff --git a/functional_tests.go b/functional_tests.go index 2a18e8a86..3e4b0010c 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -46,6 +46,7 @@ import ( "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/encrypt" "github.com/minio/minio-go/v7/pkg/notification" + "github.com/minio/minio-go/v7/pkg/tags" ) const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" @@ -1536,14 +1537,26 @@ func testObjectTaggingWithVersioning() { }) tagsV1 := map[string]string{"key1": "val1"} - err = c.PutObjectTagging(context.Background(), bucketName, objectName, tagsV1, minio.PutObjectTaggingOptions{VersionID: versions[0].VersionID}) + t1, err := tags.MapToObjectTags(tagsV1) + if err != nil { + logError(testName, function, args, startTime, "", "PutObjectTagging (1) failed", err) + return + } + + err = c.PutObjectTagging(context.Background(), bucketName, objectName, t1, minio.PutObjectTaggingOptions{VersionID: versions[0].VersionID}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectTagging (1) failed", err) return } tagsV2 := map[string]string{"key2": "val2"} - err = c.PutObjectTagging(context.Background(), bucketName, objectName, tagsV2, minio.PutObjectTaggingOptions{VersionID: versions[1].VersionID}) + t2, err := tags.MapToObjectTags(tagsV2) + if err != nil { + logError(testName, function, args, startTime, "", "PutObjectTagging (1) failed", err) + return + } + + err = c.PutObjectTagging(context.Background(), bucketName, objectName, t2, minio.PutObjectTaggingOptions{VersionID: versions[1].VersionID}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectTagging (2) failed", err) return @@ -1567,7 +1580,7 @@ func testObjectTaggingWithVersioning() { return } - if !tagsEqual(tagsV1, gotTagsV1) { + if !tagsEqual(t1.ToMap(), gotTagsV1.ToMap()) { logError(testName, function, args, startTime, "", "Unexpected tags content (1)", err) return } @@ -1578,7 +1591,7 @@ func testObjectTaggingWithVersioning() { return } - if !tagsEqual(tagsV2, gotTagsV2) { + if !tagsEqual(t2.ToMap(), gotTagsV2.ToMap()) { logError(testName, function, args, startTime, "", "Unexpected tags content (2)", err) return } @@ -1596,7 +1609,7 @@ func testObjectTaggingWithVersioning() { return } - if len(emptyTags) != 0 { + if len(emptyTags.ToMap()) != 0 { logError(testName, function, args, startTime, "", "Unexpected tags content (2)", err) return } diff --git a/pkg/lifecycle/lifecycle.go b/pkg/lifecycle/lifecycle.go index 2812e69c9..19a12c124 100644 --- a/pkg/lifecycle/lifecycle.go +++ b/pkg/lifecycle/lifecycle.go @@ -25,8 +25,22 @@ import ( // AbortIncompleteMultipartUpload structure, not supported yet on MinIO type AbortIncompleteMultipartUpload struct { - XMLName xml.Name `xml:"AbortIncompleteMultipartUpload,omitempty" json:"-"` - DaysAfterInitiation int64 `xml:"DaysAfterInitiation,omitempty" json:"DaysAfterInitiation,omitempty"` + XMLName xml.Name `xml:"AbortIncompleteMultipartUpload,omitempty" json:"-"` + DaysAfterInitiation ExpirationDays `xml:"DaysAfterInitiation,omitempty" json:"DaysAfterInitiation,omitempty"` +} + +// IsDaysNull returns true if days field is null +func (n AbortIncompleteMultipartUpload) IsDaysNull() bool { + return n.DaysAfterInitiation == ExpirationDays(0) +} + +// MarshalXML if days after initiation is set to non-zero value +func (n AbortIncompleteMultipartUpload) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if n.IsDaysNull() { + return nil + } + type abortIncompleteMultipartUploadWrapper AbortIncompleteMultipartUpload + return e.EncodeElement(abortIncompleteMultipartUploadWrapper(n), start) } // NoncurrentVersionExpiration - Specifies when noncurrent object versions expire. diff --git a/pkg/tags/tags.go b/pkg/tags/tags.go index 6ef5628e9..65ba38b10 100644 --- a/pkg/tags/tags.go +++ b/pkg/tags/tags.go @@ -255,6 +255,18 @@ func (tags Tags) ToMap() map[string]string { return tags.TagSet.toMap() } +// MapToObjectTags converts an input map of key and value into +// *Tags data structure with validation. +func MapToObjectTags(tagMap map[string]string) (*Tags, error) { + return NewTags(tagMap, true) +} + +// MapToBucketTags converts an input map of key and value into +// *Tags data structure with validation. +func MapToBucketTags(tagMap map[string]string) (*Tags, error) { + return NewTags(tagMap, false) +} + // NewTags creates Tags from tagMap, If isObject is set, it validates for object tags. func NewTags(tagMap map[string]string, isObject bool) (*Tags, error) { tagging := &Tags{ From 16b4b137ee4b439df10c224f8d34ac2ada373683 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 16 Jul 2020 17:22:36 -0700 Subject: [PATCH 189/215] ignore transition in XML if not set --- pkg/lifecycle/lifecycle.go | 42 +++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/pkg/lifecycle/lifecycle.go b/pkg/lifecycle/lifecycle.go index 19a12c124..0a6f822af 100644 --- a/pkg/lifecycle/lifecycle.go +++ b/pkg/lifecycle/lifecycle.go @@ -99,15 +99,34 @@ func (tag Tag) IsEmpty() bool { // Transition structure - transition details of lifecycle configuration type Transition struct { - XMLName xml.Name `xml:"Transition" json:"-"` - TransitionDate *time.Time `xml:"Date,omitempty" json:"Date,omitempty"` - StorageClass string `xml:"StorageClass,omitempty" json:"StorageClass,omitempty"` - TransitionInDays int `xml:"Days,omitempty" json:"Days,omitempty"` + XMLName xml.Name `xml:"Transition" json:"-"` + Date ExpirationDate `xml:"Date,omitempty" json:"Date,omitempty"` + StorageClass string `xml:"StorageClass,omitempty" json:"StorageClass,omitempty"` + Days ExpirationDays `xml:"Days,omitempty" json:"Days,omitempty"` } -// IsSet is transition set -func (t Transition) IsSet() bool { - return t.TransitionDate != nil && !t.TransitionDate.IsZero() || t.TransitionInDays > 0 +// IsDaysNull returns true if days field is null +func (t Transition) IsDaysNull() bool { + return t.Days == ExpirationDays(0) +} + +// IsDateNull returns true if date field is null +func (t Transition) IsDateNull() bool { + return t.Date.Time.IsZero() +} + +// IsNull returns true if both date and days fields are null +func (t Transition) IsNull() bool { + return t.IsDaysNull() && t.IsDateNull() +} + +// MarshalXML is transition is non null +func (t Transition) MarshalXML(en *xml.Encoder, startElement xml.StartElement) error { + if t.IsNull() { + return nil + } + type transitionWrapper Transition + return en.EncodeElement(transitionWrapper(t), startElement) } // And And Rule for LifecycleTag, to be used in LifecycleRuleFilter @@ -218,6 +237,15 @@ func (e Expiration) IsNull() bool { return e.IsDaysNull() && e.IsDateNull() } +// MarshalXML is expiration is non null +func (e Expiration) MarshalXML(en *xml.Encoder, startElement xml.StartElement) error { + if e.IsNull() { + return nil + } + type expirationWrapper Expiration + return en.EncodeElement(expirationWrapper(e), startElement) +} + // Rule represents a single rule in lifecycle configuration type Rule struct { XMLName xml.Name `xml:"Rule,omitempty" json:"-"` From 36db8b2799b43905366788913770fb51e583b4a8 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 16 Jul 2020 21:20:11 -0700 Subject: [PATCH 190/215] fix: more API refactors, remove all redundant functions (#1342) --- api-remove.go | 49 +- api.go | 108 +---- api_unit_test.go | 5 +- bucket-cache_test.go | 6 +- core.go | 4 +- core_test.go | 62 +-- functional_tests.go | 1045 +++++++++++++++++++----------------------- transport.go | 2 +- 8 files changed, 552 insertions(+), 729 deletions(-) diff --git a/api-remove.go b/api-remove.go index 28e030ced..681b3fc55 100644 --- a/api-remove.go +++ b/api-remove.go @@ -122,18 +122,21 @@ type RemoveObjectError struct { } // generateRemoveMultiObjects - generate the XML request for remove multi objects request -func generateRemoveMultiObjectsRequest(objects []ObjectVersion) []byte { - rmObjects := []deleteObject{} +func generateRemoveMultiObjectsRequest(objects []ObjectInfo) []byte { + delObjects := []deleteObject{} for _, obj := range objects { - rmObjects = append(rmObjects, deleteObject(obj)) + delObjects = append(delObjects, deleteObject{ + Key: obj.Key, + VersionID: obj.VersionID, + }) } - xmlBytes, _ := xml.Marshal(deleteMultiObjects{Objects: rmObjects, Quiet: true}) + xmlBytes, _ := xml.Marshal(deleteMultiObjects{Objects: delObjects, Quiet: true}) return xmlBytes } // processRemoveMultiObjectsResponse - parse the remove multi objects web service // and return the success/failure result status for each object -func processRemoveMultiObjectsResponse(body io.Reader, objects []ObjectVersion, errorCh chan<- RemoveObjectError) { +func processRemoveMultiObjectsResponse(body io.Reader, objects []ObjectInfo, errorCh chan<- RemoveObjectError) { // Parse multi delete XML response rmResult := &deleteMultiObjectsResult{} err := xmlDecoder(body, rmResult) @@ -154,16 +157,15 @@ func processRemoveMultiObjectsResponse(body io.Reader, objects []ObjectVersion, } } -// ObjectVersion points to a specific object version -type ObjectVersion struct { - Key string - VersionID string +// RemoveObjectsOptions represents options specified by user for RemoveObjects call +type RemoveObjectsOptions struct { + GovernanceBypass bool } -// RemoveObjectsWithVersions removes multiple objects from a bucket while +// RemoveObjects removes multiple objects from a bucket while // it is possible to specify objects versions which are received from // objectsCh. Remove failures are sent back via error channel. -func (c Client) RemoveObjectsWithVersions(ctx context.Context, bucketName string, objectsCh <-chan ObjectVersion, opts RemoveObjectsOptions) <-chan RemoveObjectError { +func (c Client) RemoveObjects(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, opts RemoveObjectsOptions) <-chan RemoveObjectError { errorCh := make(chan RemoveObjectError, 1) // Validate if bucket name is valid. @@ -188,7 +190,7 @@ func (c Client) RemoveObjectsWithVersions(ctx context.Context, bucketName string } // Generate and call MultiDelete S3 requests based on entries received from objectsCh -func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh <-chan ObjectVersion, errorCh chan<- RemoveObjectError, opts RemoveObjectsOptions) { +func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, errorCh chan<- RemoveObjectError, opts RemoveObjectsOptions) { maxEntries := 1000 finish := false urlValues := make(url.Values) @@ -203,7 +205,7 @@ func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh break } count := 0 - var batch []ObjectVersion + var batch []ObjectInfo // Try to gather 1000 entries for object := range objectsCh { @@ -264,27 +266,6 @@ func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh } } -// RemoveObjectsOptions represents options specified by user for RemoveObjects call -type RemoveObjectsOptions struct { - GovernanceBypass bool -} - -// RemoveObjects removes multiple objects from a bucket. -// The list of objects to remove are received from objectsCh. -// Remove failures are sent back via error channel. -func (c Client) RemoveObjects(ctx context.Context, bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError { - objectsVersionsCh := make(chan ObjectVersion) - go func() { - defer close(objectsVersionsCh) - for obj := range objectsCh { - objectsVersionsCh <- ObjectVersion{ - Key: obj, - } - } - }() - return c.RemoveObjectsWithVersions(ctx, bucketName, objectsVersionsCh, opts) -} - // RemoveIncompleteUpload aborts an partially uploaded object. func (c Client) RemoveIncompleteUpload(ctx context.Context, bucketName, objectName string) error { // Input validation. diff --git a/api.go b/api.go index d833ba7a6..f802cc609 100644 --- a/api.go +++ b/api.go @@ -96,6 +96,7 @@ type Client struct { type Options struct { Creds *credentials.Credentials Secure bool + Transport *http.Transport Region string BucketLookup BucketLookupType @@ -129,43 +130,12 @@ const ( BucketLookupPath ) -// NewV2 - instantiate minio client with Amazon S3 signature version -// '2' compatibility. -func NewV2(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { - clnt, err := privateNew(endpoint, Options{ - Creds: credentials.NewStaticV2(accessKeyID, secretAccessKey, ""), - Secure: secure, - BucketLookup: BucketLookupAuto, - }) - if err != nil { - return nil, err - } - clnt.overrideSignerType = credentials.SignatureV2 - return clnt, nil -} - -// NewV4 - instantiate minio client with Amazon S3 signature version -// '4' compatibility. -func NewV4(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { - clnt, err := privateNew(endpoint, Options{ - Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), - Secure: secure, - BucketLookup: BucketLookupAuto, - }) - if err != nil { - return nil, err +// New - instantiate minio client with options +func New(endpoint string, opts *Options) (*Client, error) { + if opts == nil { + return nil, errors.New("no options provided") } - clnt.overrideSignerType = credentials.SignatureV4 - return clnt, nil -} - -// New - instantiate minio client, adds automatic verification of signature. -func New(endpoint, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { - clnt, err := privateNew(endpoint, Options{ - Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), - Secure: secure, - BucketLookup: BucketLookupAuto, - }) + clnt, err := privateNew(endpoint, opts) if err != nil { return nil, err } @@ -177,40 +147,8 @@ func New(endpoint, accessKeyID, secretAccessKey string, secure bool) (*Client, e if s3utils.IsAmazonEndpoint(*clnt.endpointURL) { clnt.overrideSignerType = credentials.SignatureV4 } - return clnt, nil -} -// NewWithCredentials - instantiate minio client with credentials provider -// for retrieving credentials from various credentials provider such as -// IAM, File, Env etc. -func NewWithCredentials(endpoint string, creds *credentials.Credentials, secure bool, region string) (*Client, error) { - return privateNew(endpoint, Options{ - Creds: creds, - Secure: secure, - BucketLookup: BucketLookupAuto, - Region: region, - }) -} - -// NewWithRegion - instantiate minio client, with region configured. Unlike New(), -// NewWithRegion avoids bucket-location lookup operations and it is slightly faster. -// Use this function when if your application deals with single region. -func NewWithRegion(endpoint, accessKeyID, secretAccessKey string, secure bool, region string) (*Client, error) { - creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "") - return privateNew(endpoint, Options{ - Creds: creds, - Secure: secure, - Region: region, - BucketLookup: BucketLookupAuto, - }) -} - -// NewWithOptions - instantiate minio client with options -func NewWithOptions(endpoint string, opts *Options) (*Client, error) { - if opts == nil { - return nil, errors.New("no options provided") - } - return privateNew(endpoint, *opts) + return clnt, nil } // EndpointURL returns the URL of the S3 endpoint. @@ -302,7 +240,7 @@ func (c *Client) redirectHeaders(req *http.Request, via []*http.Request) error { return nil } -func privateNew(endpoint string, opts Options) (*Client, error) { +func privateNew(endpoint string, opts *Options) (*Client, error) { // construct endpoint. endpointURL, err := getEndpointURL(endpoint, opts.Secure) if err != nil { @@ -328,9 +266,12 @@ func privateNew(endpoint string, opts Options) (*Client, error) { // Save endpoint URL, user agent for future uses. clnt.endpointURL = endpointURL - transport, err := DefaultTransport(opts.Secure) - if err != nil { - return nil, err + transport := opts.Transport + if transport == nil { + transport, err = DefaultTransport(opts.Secure) + if err != nil { + return nil, err + } } // Instantiate http client and bucket location cache. @@ -377,27 +318,6 @@ func (c *Client) SetAppInfo(appName string, appVersion string) { } } -// SetCustomTransport - set new custom transport. -func (c *Client) SetCustomTransport(customHTTPTransport http.RoundTripper) { - // Set this to override default transport - // ``http.DefaultTransport``. - // - // This transport is usually needed for debugging OR to add your - // own custom TLS certificates on the client transport, for custom - // CA's and certs which are not part of standard certificate - // authority follow this example :- - // - // tr := &http.Transport{ - // TLSClientConfig: &tls.Config{RootCAs: pool}, - // DisableCompression: true, - // } - // api.SetCustomTransport(tr) - // - if c.httpClient != nil { - c.httpClient.Transport = customHTTPTransport - } -} - // TraceOn - enable HTTP tracing. func (c *Client) TraceOn(outputStream io.Writer) { // if outputStream is nil then default to os.Stdout. diff --git a/api_unit_test.go b/api_unit_test.go index 4dd0162d9..fb6977d8b 100644 --- a/api_unit_test.go +++ b/api_unit_test.go @@ -200,7 +200,10 @@ func TestMakeTargetURL(t *testing.T) { for i, testCase := range testCases { // Initialize a MinIO client - c, _ := New(testCase.addr, "foo", "bar", testCase.secure) + c, _ := New(testCase.addr, &Options{ + Creds: credentials.NewStaticV4("foo", "bar", ""), + Secure: testCase.secure, + }) isVirtualHost := c.isVirtualHostStyleRequest(*c.endpointURL, testCase.bucketName) u, err := c.makeTargetURL(testCase.bucketName, testCase.objectName, testCase.bucketLocation, isVirtualHost, testCase.queryValues) // Check the returned error diff --git a/bucket-cache_test.go b/bucket-cache_test.go index 44c13b115..d70a08515 100644 --- a/bucket-cache_test.go +++ b/bucket-cache_test.go @@ -227,8 +227,10 @@ func TestGetBucketLocationRequest(t *testing.T) { client := &Client{} var err error if testCase.info.endPoint != "" { - - client, err = New(testCase.info.endPoint, testCase.info.accessKey, testCase.info.secretKey, testCase.info.enableInsecure) + client, err = New(testCase.info.endPoint, &Options{ + Creds: credentials.NewStaticV4(testCase.info.accessKey, testCase.info.secretKey, ""), + Secure: testCase.info.enableInsecure, + }) if err != nil { t.Fatalf("Test %d: Failed to create new Client: %s", i+1, err.Error()) } diff --git a/core.go b/core.go index f6073225a..954e1a1bb 100644 --- a/core.go +++ b/core.go @@ -33,9 +33,9 @@ type Core struct { // NewCore - Returns new initialized a Core client, this CoreClient should be // only used under special conditions such as need to access lower primitives // and being able to use them to write your own wrappers. -func NewCore(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Core, error) { +func NewCore(endpoint string, opts *Options) (*Core, error) { var s3Client Core - client, err := NewV4(endpoint, accessKeyID, secretAccessKey, secure) + client, err := New(endpoint, opts) if err != nil { return nil, err } diff --git a/core_test.go b/core_test.go index 61a01540d..0279f6bb3 100644 --- a/core_test.go +++ b/core_test.go @@ -29,6 +29,8 @@ import ( "time" "math/rand" + + "github.com/minio/minio-go/v7/pkg/credentials" ) const ( @@ -75,10 +77,10 @@ func TestGetObjectCore(t *testing.T) { // Instantiate new minio core client object. c, err := NewCore( os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) + &Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableSecurity)), + }) if err != nil { t.Fatal("Error:", err) } @@ -272,10 +274,10 @@ func TestGetObjectContentEncoding(t *testing.T) { // Instantiate new minio core client object. c, err := NewCore( os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) + &Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableSecurity)), + }) if err != nil { t.Fatal("Error:", err) } @@ -345,10 +347,10 @@ func TestGetBucketPolicy(t *testing.T) { // Instantiate new minio client object. c, err := NewCore( os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) + &Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableSecurity)), + }) if err != nil { t.Fatal("Error:", err) } @@ -408,10 +410,10 @@ func TestCoreCopyObject(t *testing.T) { // Instantiate new minio client object. c, err := NewCore( os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) + &Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableSecurity)), + }) if err != nil { t.Fatal("Error:", err) } @@ -531,10 +533,10 @@ func TestCoreCopyObjectPart(t *testing.T) { // Instantiate new minio client object. c, err := NewCore( os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) + &Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableSecurity)), + }) if err != nil { t.Fatal("Error:", err) } @@ -684,13 +686,12 @@ func TestCorePutObject(t *testing.T) { // Instantiate new minio client object. c, err := NewCore( os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) + &Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableSecurity)), + }) if err != nil { - t.Error("Error:", err) - return + t.Fatal("Error:", err) } // Enable tracing, write to stderr. @@ -775,11 +776,12 @@ func TestCoreGetObjectMetadata(t *testing.T) { core, err := NewCore( os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity))) + &Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableSecurity)), + }) if err != nil { - log.Fatalln(err) + t.Fatal(err) } // Generate a new random bucket name. diff --git a/functional_tests.go b/functional_tests.go index 3e4b0010c..a4502c02b 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -44,6 +44,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/encrypt" "github.com/minio/minio-go/v7/pkg/notification" "github.com/minio/minio-go/v7/pkg/tags" @@ -337,12 +338,11 @@ func testMakeBucketError() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client creation failed", err) return @@ -393,12 +393,11 @@ func testMetadataSizeLimit() { rand.Seed(startTime.Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client creation failed", err) return @@ -473,12 +472,11 @@ func testMakeBucketRegions() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client creation failed", err) return @@ -540,12 +538,11 @@ func testPutObjectReadAt() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -639,12 +636,11 @@ func testListObjectVersions() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -759,12 +755,11 @@ func testStatObjectWithVersioning() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -873,12 +868,11 @@ func testGetObjectWithVersioning() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -1013,12 +1007,11 @@ func testCopyObjectWithVersioning() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -1151,12 +1144,11 @@ func testComposeObjectWithVersioning() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -1292,12 +1284,11 @@ func testRemoveObjectWithVersioning() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -1378,12 +1369,11 @@ func testRemoveObjectsWithVersioning() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -1421,23 +1411,21 @@ func testRemoveObjectsWithVersioning() { return } - objectsVersions := make(chan minio.ObjectVersion) + objectsVersions := make(chan minio.ObjectInfo) go func() { - objectsVersionsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) + objectsVersionsInfo := c.ListObjects(context.Background(), bucketName, + minio.ListObjectsOptions{WithVersions: true, Recursive: true}) for info := range objectsVersionsInfo { if info.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) return } - objectsVersions <- minio.ObjectVersion{ - Key: info.Key, - VersionID: info.VersionID, - } + objectsVersions <- info } close(objectsVersions) }() - removeErrors := c.RemoveObjectsWithVersions(context.Background(), bucketName, objectsVersions, minio.RemoveObjectsOptions{}) + removeErrors := c.RemoveObjects(context.Background(), bucketName, objectsVersions, minio.RemoveObjectsOptions{}) if err != nil { logError(testName, function, args, startTime, "", "DeleteObjects call failed", err) return @@ -1476,12 +1464,11 @@ func testObjectTaggingWithVersioning() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -1644,12 +1631,11 @@ func testPutObjectWithMetadata() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -1750,12 +1736,11 @@ func testPutObjectWithContentLanguage() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -1824,12 +1809,11 @@ func testPutObjectStreaming() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -1900,12 +1884,11 @@ func testGetObjectSeekEnd() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -2027,12 +2010,11 @@ func testGetObjectClosedTwice() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -2118,12 +2100,11 @@ func testRemoveObjectsContext() { rand.Seed(time.Now().Unix()) // Instantiate new minio client. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -2149,17 +2130,21 @@ func testRemoveObjectsContext() { // Multi remove of 20 objects. nrObjects := 20 - objectsCh := make(chan string) + objectsCh := make(chan minio.ObjectInfo) go func() { defer close(objectsCh) for i := 0; i < nrObjects; i++ { objectName := "sample" + strconv.Itoa(i) + ".txt" - _, err = c.PutObject(context.Background(), bucketName, objectName, r, 8, minio.PutObjectOptions{ContentType: "application/octet-stream"}) + info, err := c.PutObject(context.Background(), bucketName, objectName, r, 8, + minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) continue } - objectsCh <- objectName + objectsCh <- minio.ObjectInfo{ + Key: info.Key, + VersionID: info.VersionID, + } } }() // Set context to cancel in 1 nanosecond. @@ -2213,13 +2198,11 @@ func testRemoveMultipleObjects() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -2247,19 +2230,23 @@ func testRemoveMultipleObjects() { // Multi remove of 1100 objects nrObjects := 200 - objectsCh := make(chan string) + objectsCh := make(chan minio.ObjectInfo) go func() { defer close(objectsCh) // Upload objects and send them to objectsCh for i := 0; i < nrObjects; i++ { objectName := "sample" + strconv.Itoa(i) + ".txt" - _, err = c.PutObject(context.Background(), bucketName, objectName, r, 8, minio.PutObjectOptions{ContentType: "application/octet-stream"}) + info, err := c.PutObject(context.Background(), bucketName, objectName, r, 8, + minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) continue } - objectsCh <- objectName + objectsCh <- minio.ObjectInfo{ + Key: info.Key, + VersionID: info.VersionID, + } } }() @@ -2301,12 +2288,11 @@ func testFPutObjectMultipart() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -2411,12 +2397,11 @@ func testFPutObject() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -2585,12 +2570,11 @@ func testFPutObjectContext() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -2691,12 +2675,11 @@ func testFPutObjectContextV2() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -2796,13 +2779,13 @@ func testPutObjectContext() { "objectName": "", "opts": "", } + // Instantiate new minio client object. - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -2874,12 +2857,11 @@ func testGetObjectReadSeekFunctional() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -3044,12 +3026,11 @@ func testGetObjectReadAtFunctional() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -3224,12 +3205,11 @@ func testGetObjectReadAtWhenEOFWasReached() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -3348,12 +3328,11 @@ func testPresignedPostPolicy() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -3564,12 +3543,11 @@ func testCopyObject() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -3757,12 +3735,11 @@ func testSSECEncryptedGetObjectReadSeekFunctional() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -3940,12 +3917,11 @@ func testSSES3EncryptedGetObjectReadSeekFunctional() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -4121,12 +4097,11 @@ func testSSECEncryptedGetObjectReadAtFunctional() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -4307,12 +4282,11 @@ func testSSES3EncryptedGetObjectReadAtFunctional() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -4494,12 +4468,11 @@ func testSSECEncryptionPutGet() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -4608,12 +4581,11 @@ func testSSECEncryptionFPut() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -4735,12 +4707,11 @@ func testSSES3EncryptionPutGet() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -4847,12 +4818,11 @@ func testSSES3EncryptionFPut() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -4979,12 +4949,11 @@ func testBucketNotification() { // Seed random based on current time. rand.Seed(time.Now().Unix()) - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -5074,12 +5043,11 @@ func testFunctional() { // Seed random based on current time. rand.Seed(time.Now().Unix()) - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, nil, startTime, "", "MinIO client object creation failed", err) return @@ -5700,13 +5668,11 @@ func testGetObjectModified() { args := map[string]interface{}{} // Instantiate new minio client object. - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -5801,12 +5767,11 @@ func testPutObjectUploadSeekedObject() { } // Instantiate new minio client object. - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -5935,12 +5900,11 @@ func testMakeBucketErrorV2() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return @@ -5996,12 +5960,11 @@ func testGetObjectClosedTwiceV2() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return @@ -6091,12 +6054,11 @@ func testFPutObjectV2() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return @@ -6261,12 +6223,11 @@ func testMakeBucketRegionsV2() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return @@ -6324,12 +6285,11 @@ func testGetObjectReadSeekFunctionalV2() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return @@ -6483,12 +6443,11 @@ func testGetObjectReadAtFunctionalV2() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return @@ -6648,12 +6607,11 @@ func testCopyObjectV2() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return @@ -6876,12 +6834,11 @@ func testComposeObjectErrorCasesV2() { args := map[string]interface{}{} // Instantiate new minio client object - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return @@ -6976,12 +6933,11 @@ func testCompose10KSourcesV2() { args := map[string]interface{}{} // Instantiate new minio client object - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return @@ -6998,12 +6954,11 @@ func testEncryptedEmptyObject() { args := map[string]interface{}{} // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return @@ -7266,12 +7221,11 @@ func testUnencryptedToSSECCopyObject() { args := map[string]interface{}{} // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return @@ -7294,12 +7248,11 @@ func testUnencryptedToSSES3CopyObject() { args := map[string]interface{}{} // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return @@ -7322,12 +7275,11 @@ func testUnencryptedToUnencryptedCopyObject() { args := map[string]interface{}{} // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return @@ -7349,12 +7301,11 @@ func testEncryptedSSECToSSECCopyObject() { args := map[string]interface{}{} // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return @@ -7377,12 +7328,11 @@ func testEncryptedSSECToSSES3CopyObject() { args := map[string]interface{}{} // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return @@ -7405,12 +7355,11 @@ func testEncryptedSSECToUnencryptedCopyObject() { args := map[string]interface{}{} // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return @@ -7433,12 +7382,11 @@ func testEncryptedSSES3ToSSECCopyObject() { args := map[string]interface{}{} // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return @@ -7461,12 +7409,11 @@ func testEncryptedSSES3ToSSES3CopyObject() { args := map[string]interface{}{} // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return @@ -7489,12 +7436,11 @@ func testEncryptedSSES3ToUnencryptedCopyObject() { args := map[string]interface{}{} // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return @@ -7517,12 +7463,11 @@ func testEncryptedCopyObjectV2() { args := map[string]interface{}{} // Instantiate new minio client object - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return @@ -7544,12 +7489,11 @@ func testDecryptedCopyObject() { args := map[string]interface{}{} // Instantiate new minio client object - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return @@ -7602,12 +7546,11 @@ func testSSECMultipartEncryptedToSSECCopyObjectPart() { args := map[string]interface{}{} // Instantiate new minio client object - client, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + client, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return @@ -7774,12 +7717,11 @@ func testSSECEncryptedToSSECCopyObjectPart() { args := map[string]interface{}{} // Instantiate new minio client object - client, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + client, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return @@ -7934,12 +7876,11 @@ func testSSECEncryptedToUnencryptedCopyPart() { args := map[string]interface{}{} // Instantiate new minio client object - client, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + client, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return @@ -8093,12 +8034,11 @@ func testSSECEncryptedToSSES3CopyObjectPart() { args := map[string]interface{}{} // Instantiate new minio client object - client, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + client, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return @@ -8255,12 +8195,11 @@ func testUnencryptedToSSECCopyObjectPart() { args := map[string]interface{}{} // Instantiate new minio client object - client, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + client, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return @@ -8412,12 +8351,11 @@ func testUnencryptedToUnencryptedCopyPart() { args := map[string]interface{}{} // Instantiate new minio client object - client, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + client, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return @@ -8565,12 +8503,11 @@ func testUnencryptedToSSES3CopyObjectPart() { args := map[string]interface{}{} // Instantiate new minio client object - client, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + client, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return @@ -8720,12 +8657,11 @@ func testSSES3EncryptedToSSECCopyObjectPart() { args := map[string]interface{}{} // Instantiate new minio client object - client, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + client, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return @@ -8878,12 +8814,11 @@ func testSSES3EncryptedToUnencryptedCopyPart() { args := map[string]interface{}{} // Instantiate new minio client object - client, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + client, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return @@ -9032,12 +8967,11 @@ func testSSES3EncryptedToSSES3CopyObjectPart() { args := map[string]interface{}{} // Instantiate new minio client object - client, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + client, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return @@ -9187,12 +9121,11 @@ func testUserMetadataCopying() { args := map[string]interface{}{} // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -9368,12 +9301,11 @@ func testUserMetadataCopyingV2() { args := map[string]interface{}{} // Instantiate new minio client object - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return @@ -9391,12 +9323,11 @@ func testStorageClassMetadataPutObject() { testName := getFuncName() // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return @@ -9481,12 +9412,11 @@ func testStorageClassInvalidMetadataPutObject() { testName := getFuncName() // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return @@ -9526,12 +9456,11 @@ func testStorageClassMetadataCopyObject() { testName := getFuncName() // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return @@ -9656,12 +9585,11 @@ func testPutObjectNoLengthV2() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return @@ -9736,12 +9664,11 @@ func testPutObjectsUnknownV2() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return @@ -9831,12 +9758,11 @@ func testPutObject0ByteV2() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return @@ -9897,12 +9823,11 @@ func testComposeObjectErrorCases() { args := map[string]interface{}{} // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -9920,12 +9845,11 @@ func testCompose10KSources() { args := map[string]interface{}{} // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return @@ -9946,12 +9870,11 @@ func testFunctionalV2() { // Seed random based on current time. rand.Seed(time.Now().Unix()) - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return @@ -10394,12 +10317,11 @@ func testGetObjectContext() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return @@ -10502,12 +10424,11 @@ func testFGetObjectContext() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return @@ -10599,12 +10520,11 @@ func testGetObjectACLContext() { } // Instantiate new minio client object. - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return @@ -10762,12 +10682,11 @@ func testPutObjectContextV2() { "opts": "", } // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return @@ -10844,12 +10763,11 @@ func testGetObjectContextV2() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return @@ -10950,12 +10868,11 @@ func testFGetObjectContextV2() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return @@ -11043,12 +10960,11 @@ func testListObjects() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return @@ -11145,12 +11061,11 @@ func testRemoveObjects() { rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return @@ -11197,7 +11112,7 @@ func testRemoveObjects() { log.Fatalln(err) } - objectsCh := make(chan string) + objectsCh := make(chan minio.ObjectInfo) // Send object names that are needed to be removed to objectsCh go func() { defer close(objectsCh) @@ -11206,7 +11121,7 @@ func testRemoveObjects() { if object.Err != nil { log.Fatalln(object.Err) } - objectsCh <- object.Key + objectsCh <- object } }() @@ -11219,7 +11134,7 @@ func testRemoveObjects() { } } - objectsCh1 := make(chan string) + objectsCh1 := make(chan minio.ObjectInfo) // Send object names that are needed to be removed to objectsCh go func() { @@ -11229,7 +11144,7 @@ func testRemoveObjects() { if object.Err != nil { log.Fatalln(object.Err) } - objectsCh1 <- object.Key + objectsCh1 <- object } }() diff --git a/transport.go b/transport.go index 4a1e17488..d5ad15b8b 100644 --- a/transport.go +++ b/transport.go @@ -41,7 +41,7 @@ func mustGetSystemCertPool() *x509.CertPool { // DefaultTransport - this default transport is similar to // http.DefaultTransport but with additional param DisableCompression // is set to true to avoid decompressing content with 'gzip' encoding. -var DefaultTransport = func(secure bool) (http.RoundTripper, error) { +var DefaultTransport = func(secure bool) (*http.Transport, error) { tr := &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ From 18af734a86f269a97cd59717af44baf63bb63f19 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Fri, 17 Jul 2020 04:23:45 +0000 Subject: [PATCH 191/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index f802cc609..6f245140e 100644 --- a/api.go +++ b/api.go @@ -108,7 +108,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v7.0.0" + libraryVersion = "v7.0.1" ) // User Agent should always following the below style. From ac7716b311b1705875436f158b2abc887e538e1c Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 16 Jul 2020 21:40:48 -0700 Subject: [PATCH 192/215] add transport to be roundtripper --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 6f245140e..d5ec750ac 100644 --- a/api.go +++ b/api.go @@ -96,7 +96,7 @@ type Client struct { type Options struct { Creds *credentials.Credentials Secure bool - Transport *http.Transport + Transport http.RoundTripper Region string BucketLookup BucketLookupType From c693f76f982b49d47514163d7e182e0f2a41553a Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Fri, 17 Jul 2020 04:43:20 +0000 Subject: [PATCH 193/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index d5ec750ac..cd14a5fda 100644 --- a/api.go +++ b/api.go @@ -108,7 +108,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v7.0.1" + libraryVersion = "v7.0.2" ) // User Agent should always following the below style. From 19a9c783bfe01da6692f1d31cad47683ead55a73 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 17 Jul 2020 11:18:55 -0700 Subject: [PATCH 194/215] fix: update API docs and examples --- README.md | 23 ++++++++++++++++------ examples/minio/listenbucketnotification.go | 6 +++++- examples/s3/bucketexists.go | 6 +++++- examples/s3/composeobject.go | 6 +++++- examples/s3/copyobject-with-new-tags.go | 6 +++++- examples/s3/copyobject.go | 6 +++++- examples/s3/disableversioning.go | 6 +++++- examples/s3/enableversioning.go | 6 +++++- examples/s3/fgetobject.go | 6 +++++- examples/s3/fputencrypted-object.go | 6 +++++- examples/s3/fputobject.go | 6 +++++- examples/s3/getbucketencryption.go | 6 +++++- examples/s3/getbucketlifecycle.go | 6 +++++- examples/s3/getbucketnotification.go | 6 +++++- examples/s3/getbucketpolicy.go | 6 +++++- examples/s3/getbucketreplication.go | 6 +++++- examples/s3/getbuckettagging.go | 6 +++++- examples/s3/getbucketversioning.go | 6 +++++- examples/s3/getobject-client-encryption.go | 6 +++++- examples/s3/getobjecttagging.go | 6 +++++- examples/s3/listbuckets.go | 6 +++++- examples/s3/listincompleteuploads.go | 6 +++++- examples/s3/listobjects-N.go | 6 +++++- examples/s3/listobjects.go | 6 +++++- examples/s3/listobjectsV2.go | 6 +++++- examples/s3/listobjectsV2WithMetadata.go | 6 +++++- examples/s3/listobjectversions.go | 6 +++++- examples/s3/makebucket.go | 6 +++++- examples/s3/presignedgetobject.go | 6 +++++- examples/s3/presignedheadobject.go | 6 +++++- examples/s3/presignedpostpolicy.go | 6 +++++- examples/s3/presignedputobject.go | 6 +++++- examples/s3/put-encrypted-object.go | 6 +++++- examples/s3/putobject-client-encryption.go | 6 +++++- examples/s3/putobject-getobject-sse.go | 6 +++++- examples/s3/putobject-progress.go | 6 +++++- examples/s3/putobject-s3-accelerate.go | 6 +++++- examples/s3/putobject-streaming.go | 6 +++++- examples/s3/putobject-with-tags.go | 6 +++++- examples/s3/putobject.go | 6 +++++- examples/s3/putobjecttagging.go | 6 +++++- examples/s3/removeallbucketnotification.go | 6 +++++- examples/s3/removebucket.go | 6 +++++- examples/s3/removebucketencryption.go | 6 +++++- examples/s3/removebucketreplication.go | 6 +++++- examples/s3/removebuckettagging.go | 6 +++++- examples/s3/removeincompleteupload.go | 6 +++++- examples/s3/removeobject.go | 6 +++++- examples/s3/removeobjects.go | 6 +++++- examples/s3/removeobjecttagging.go | 6 +++++- examples/s3/setbucketencryption.go | 6 +++++- examples/s3/setbucketlifecycle.go | 6 +++++- examples/s3/setbucketnotification.go | 6 +++++- examples/s3/setbucketpolicy.go | 6 +++++- examples/s3/setbucketreplication.go | 6 +++++- examples/s3/setbuckettagging.go | 6 +++++- examples/s3/setobjectlockconfig.go | 6 +++++- examples/s3/statobject.go | 6 +++++- 58 files changed, 302 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index c740f6ed1..b0d85237e 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,10 @@ MinIO client requires the following four parameters specified to connect to an A package main import ( - "github.com/minio/minio-go/v7" "log" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -37,7 +39,10 @@ func main() { useSSL := true // Initialize minio client object. - minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL) + minioClient, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: useSSL, + }) if err != nil { log.Fatalln(err) } @@ -56,8 +61,11 @@ We will use the MinIO server running at [https://play.min.io](https://play.min.i package main import ( - "github.com/minio/minio-go/v7" + "context" "log" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -67,7 +75,10 @@ func main() { useSSL := true // Initialize minio client object. - minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL) + minioClient, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: useSSL, + }) if err != nil { log.Fatalln(err) } @@ -76,7 +87,7 @@ func main() { bucketName := "mymusic" location := "us-east-1" - err = minioClient.MakeBucket(bucketName, location) + err = minioClient.MakeBucket(context.Background(), bucketName, MakeBucketOptions{Region: location}) if err != nil { // Check to see if we already own this bucket (which happens if you run this twice) exists, errBucketExists := minioClient.BucketExists(bucketName) @@ -95,7 +106,7 @@ func main() { contentType := "application/zip" // Upload the zip file with FPutObject - n, err := minioClient.FPutObject(bucketName, objectName, filePath, minio.PutObjectOptions{ContentType:contentType}) + n, err := minioClient.FPutObject(context.Background(), bucketName, objectName, filePath, minio.PutObjectOptions{ContentType: contentType}) if err != nil { log.Fatalln(err) } diff --git a/examples/minio/listenbucketnotification.go b/examples/minio/listenbucketnotification.go index 77eca42db..92b5c67c4 100644 --- a/examples/minio/listenbucketnotification.go +++ b/examples/minio/listenbucketnotification.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - minioClient, err := minio.New("play.min.io", "YOUR-ACCESS", "YOUR-SECRET", true) + minioClient, err := minio.New("play.min.io", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/bucketexists.go b/examples/s3/bucketexists.go index 5f98e0838..eafbb5568 100644 --- a/examples/s3/bucketexists.go +++ b/examples/s3/bucketexists.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/composeobject.go b/examples/s3/composeobject.go index 5798ca34e..2b9467a0f 100644 --- a/examples/s3/composeobject.go +++ b/examples/s3/composeobject.go @@ -24,6 +24,7 @@ import ( "log" minio "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/encrypt" ) @@ -36,7 +37,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/copyobject-with-new-tags.go b/examples/s3/copyobject-with-new-tags.go index 2cb3dccc6..60c2ac0b1 100644 --- a/examples/s3/copyobject-with-new-tags.go +++ b/examples/s3/copyobject-with-new-tags.go @@ -25,6 +25,7 @@ import ( "time" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -36,7 +37,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/copyobject.go b/examples/s3/copyobject.go index 6499b9870..989ac2ee0 100644 --- a/examples/s3/copyobject.go +++ b/examples/s3/copyobject.go @@ -25,6 +25,7 @@ import ( "time" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -36,7 +37,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/disableversioning.go b/examples/s3/disableversioning.go index 711d666e8..a0854d45d 100644 --- a/examples/s3/disableversioning.go +++ b/examples/s3/disableversioning.go @@ -24,6 +24,7 @@ import ( "log" minio "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/enableversioning.go b/examples/s3/enableversioning.go index 4de839852..6876d444f 100644 --- a/examples/s3/enableversioning.go +++ b/examples/s3/enableversioning.go @@ -24,6 +24,7 @@ import ( "log" minio "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/fgetobject.go b/examples/s3/fgetobject.go index 33305657c..cbd17f798 100644 --- a/examples/s3/fgetobject.go +++ b/examples/s3/fgetobject.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/fputencrypted-object.go b/examples/s3/fputencrypted-object.go index 834f79dcc..9bea03719 100644 --- a/examples/s3/fputencrypted-object.go +++ b/examples/s3/fputencrypted-object.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/encrypt" ) @@ -36,7 +37,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/fputobject.go b/examples/s3/fputobject.go index 458c09c38..bb4ffc8f0 100644 --- a/examples/s3/fputobject.go +++ b/examples/s3/fputobject.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getbucketencryption.go b/examples/s3/getbucketencryption.go index 719ead8ff..0d84f38fb 100644 --- a/examples/s3/getbucketencryption.go +++ b/examples/s3/getbucketencryption.go @@ -25,6 +25,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -36,7 +37,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getbucketlifecycle.go b/examples/s3/getbucketlifecycle.go index 7d7d701d8..c090fb927 100644 --- a/examples/s3/getbucketlifecycle.go +++ b/examples/s3/getbucketlifecycle.go @@ -27,6 +27,7 @@ import ( "strings" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -38,7 +39,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getbucketnotification.go b/examples/s3/getbucketnotification.go index 75909afea..efafac865 100644 --- a/examples/s3/getbucketnotification.go +++ b/examples/s3/getbucketnotification.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getbucketpolicy.go b/examples/s3/getbucketpolicy.go index c11e838ab..1cf2e2486 100644 --- a/examples/s3/getbucketpolicy.go +++ b/examples/s3/getbucketpolicy.go @@ -27,6 +27,7 @@ import ( "strings" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -38,7 +39,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getbucketreplication.go b/examples/s3/getbucketreplication.go index d7f9a95c0..aed17442d 100644 --- a/examples/s3/getbucketreplication.go +++ b/examples/s3/getbucketreplication.go @@ -26,6 +26,7 @@ import ( "os" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -37,7 +38,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getbuckettagging.go b/examples/s3/getbuckettagging.go index db8dd2040..9d2067e66 100644 --- a/examples/s3/getbuckettagging.go +++ b/examples/s3/getbuckettagging.go @@ -25,6 +25,7 @@ import ( "log" minio "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -36,7 +37,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getbucketversioning.go b/examples/s3/getbucketversioning.go index d30ed30e7..9b55d892d 100644 --- a/examples/s3/getbucketversioning.go +++ b/examples/s3/getbucketversioning.go @@ -25,6 +25,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -36,7 +37,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getobject-client-encryption.go b/examples/s3/getobject-client-encryption.go index f910072e7..7d75baaa8 100644 --- a/examples/s3/getobject-client-encryption.go +++ b/examples/s3/getobject-client-encryption.go @@ -26,6 +26,7 @@ import ( "path" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/sio" "golang.org/x/crypto/argon2" ) @@ -39,7 +40,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getobjecttagging.go b/examples/s3/getobjecttagging.go index 4c189377e..09674f399 100644 --- a/examples/s3/getobjecttagging.go +++ b/examples/s3/getobjecttagging.go @@ -25,6 +25,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -36,7 +37,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/listbuckets.go b/examples/s3/listbuckets.go index dd360e85d..348366218 100644 --- a/examples/s3/listbuckets.go +++ b/examples/s3/listbuckets.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/listincompleteuploads.go b/examples/s3/listincompleteuploads.go index 8dfa192ab..adcc10561 100644 --- a/examples/s3/listincompleteuploads.go +++ b/examples/s3/listincompleteuploads.go @@ -25,6 +25,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -36,7 +37,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/listobjects-N.go b/examples/s3/listobjects-N.go index 9e189bd11..0aaa9852b 100644 --- a/examples/s3/listobjects-N.go +++ b/examples/s3/listobjects-N.go @@ -24,6 +24,7 @@ import ( "fmt" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { fmt.Println(err) return diff --git a/examples/s3/listobjects.go b/examples/s3/listobjects.go index 7bcb04797..3dd0bf36c 100644 --- a/examples/s3/listobjects.go +++ b/examples/s3/listobjects.go @@ -24,6 +24,7 @@ import ( "fmt" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { fmt.Println(err) return diff --git a/examples/s3/listobjectsV2.go b/examples/s3/listobjectsV2.go index 7d92d82f9..99dfecbeb 100644 --- a/examples/s3/listobjectsV2.go +++ b/examples/s3/listobjectsV2.go @@ -24,6 +24,7 @@ import ( "fmt" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { fmt.Println(err) return diff --git a/examples/s3/listobjectsV2WithMetadata.go b/examples/s3/listobjectsV2WithMetadata.go index c7de17782..34294b171 100644 --- a/examples/s3/listobjectsV2WithMetadata.go +++ b/examples/s3/listobjectsV2WithMetadata.go @@ -24,6 +24,7 @@ import ( "fmt" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { fmt.Println(err) return diff --git a/examples/s3/listobjectversions.go b/examples/s3/listobjectversions.go index 8ce3d1806..518544466 100644 --- a/examples/s3/listobjectversions.go +++ b/examples/s3/listobjectversions.go @@ -24,6 +24,7 @@ import ( "fmt" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { fmt.Println(err) return diff --git a/examples/s3/makebucket.go b/examples/s3/makebucket.go index e64fa3f63..b136af1e0 100644 --- a/examples/s3/makebucket.go +++ b/examples/s3/makebucket.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/presignedgetobject.go b/examples/s3/presignedgetobject.go index 3a8545bd5..731727ff5 100644 --- a/examples/s3/presignedgetobject.go +++ b/examples/s3/presignedgetobject.go @@ -26,6 +26,7 @@ import ( "time" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -37,7 +38,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/presignedheadobject.go b/examples/s3/presignedheadobject.go index dacd32ed4..2d9afccd1 100644 --- a/examples/s3/presignedheadobject.go +++ b/examples/s3/presignedheadobject.go @@ -26,6 +26,7 @@ import ( "time" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -37,7 +38,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/presignedpostpolicy.go b/examples/s3/presignedpostpolicy.go index 99c223b77..e5fe1523c 100644 --- a/examples/s3/presignedpostpolicy.go +++ b/examples/s3/presignedpostpolicy.go @@ -26,6 +26,7 @@ import ( "time" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -37,7 +38,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/presignedputobject.go b/examples/s3/presignedputobject.go index c8e2d1847..fed3ce4b0 100644 --- a/examples/s3/presignedputobject.go +++ b/examples/s3/presignedputobject.go @@ -25,6 +25,7 @@ import ( "time" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -36,7 +37,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/put-encrypted-object.go b/examples/s3/put-encrypted-object.go index 720c6c0ac..e0c42f66b 100644 --- a/examples/s3/put-encrypted-object.go +++ b/examples/s3/put-encrypted-object.go @@ -25,6 +25,7 @@ import ( "os" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/encrypt" ) @@ -37,7 +38,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/putobject-client-encryption.go b/examples/s3/putobject-client-encryption.go index a033b3df4..04d6055b3 100644 --- a/examples/s3/putobject-client-encryption.go +++ b/examples/s3/putobject-client-encryption.go @@ -26,6 +26,7 @@ import ( "path" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/sio" "golang.org/x/crypto/argon2" ) @@ -39,7 +40,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/putobject-getobject-sse.go b/examples/s3/putobject-getobject-sse.go index 4c54f2c0a..2764a2034 100644 --- a/examples/s3/putobject-getobject-sse.go +++ b/examples/s3/putobject-getobject-sse.go @@ -26,6 +26,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/encrypt" ) @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - minioClient, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + minioClient, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/putobject-progress.go b/examples/s3/putobject-progress.go index 361e21e8d..82b3228b6 100644 --- a/examples/s3/putobject-progress.go +++ b/examples/s3/putobject-progress.go @@ -25,6 +25,7 @@ import ( "github.com/cheggaaa/pb" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -36,7 +37,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/putobject-s3-accelerate.go b/examples/s3/putobject-s3-accelerate.go index cc738145f..f79564217 100644 --- a/examples/s3/putobject-s3-accelerate.go +++ b/examples/s3/putobject-s3-accelerate.go @@ -25,6 +25,7 @@ import ( "os" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -36,7 +37,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/putobject-streaming.go b/examples/s3/putobject-streaming.go index 3b14b1598..3ccdbcaca 100644 --- a/examples/s3/putobject-streaming.go +++ b/examples/s3/putobject-streaming.go @@ -25,6 +25,7 @@ import ( "os" minio "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -36,7 +37,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/putobject-with-tags.go b/examples/s3/putobject-with-tags.go index 6818933ce..4b8b9b9b2 100644 --- a/examples/s3/putobject-with-tags.go +++ b/examples/s3/putobject-with-tags.go @@ -25,6 +25,7 @@ import ( "os" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -36,7 +37,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/putobject.go b/examples/s3/putobject.go index 43251c23d..7f6071a38 100644 --- a/examples/s3/putobject.go +++ b/examples/s3/putobject.go @@ -25,6 +25,7 @@ import ( "os" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -36,7 +37,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/putobjecttagging.go b/examples/s3/putobjecttagging.go index 722ae875c..8ba1c6fb1 100644 --- a/examples/s3/putobjecttagging.go +++ b/examples/s3/putobjecttagging.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/tags" ) @@ -36,7 +37,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/removeallbucketnotification.go b/examples/s3/removeallbucketnotification.go index 39a9eafd5..c6caa67d9 100644 --- a/examples/s3/removeallbucketnotification.go +++ b/examples/s3/removeallbucketnotification.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/removebucket.go b/examples/s3/removebucket.go index 12f918ce9..581776ca5 100644 --- a/examples/s3/removebucket.go +++ b/examples/s3/removebucket.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/removebucketencryption.go b/examples/s3/removebucketencryption.go index 71f1926b1..b06a93762 100644 --- a/examples/s3/removebucketencryption.go +++ b/examples/s3/removebucketencryption.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/removebucketreplication.go b/examples/s3/removebucketreplication.go index c8769f110..32d4905d2 100644 --- a/examples/s3/removebucketreplication.go +++ b/examples/s3/removebucketreplication.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/removebuckettagging.go b/examples/s3/removebuckettagging.go index 534ef7403..99a8e5db5 100644 --- a/examples/s3/removebuckettagging.go +++ b/examples/s3/removebuckettagging.go @@ -24,6 +24,7 @@ import ( "log" minio "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/removeincompleteupload.go b/examples/s3/removeincompleteupload.go index 1681b016b..cf16e0e4c 100644 --- a/examples/s3/removeincompleteupload.go +++ b/examples/s3/removeincompleteupload.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/removeobject.go b/examples/s3/removeobject.go index 9a42ba425..777835d7d 100644 --- a/examples/s3/removeobject.go +++ b/examples/s3/removeobject.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/removeobjects.go b/examples/s3/removeobjects.go index 681b0269d..2250bbf31 100644 --- a/examples/s3/removeobjects.go +++ b/examples/s3/removeobjects.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/removeobjecttagging.go b/examples/s3/removeobjecttagging.go index 47ac71d55..b359befd6 100644 --- a/examples/s3/removeobjecttagging.go +++ b/examples/s3/removeobjecttagging.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/setbucketencryption.go b/examples/s3/setbucketencryption.go index b742f1e73..81e3c05d5 100644 --- a/examples/s3/setbucketencryption.go +++ b/examples/s3/setbucketencryption.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/sse" ) @@ -36,7 +37,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/setbucketlifecycle.go b/examples/s3/setbucketlifecycle.go index a527d797c..2f4869d8d 100644 --- a/examples/s3/setbucketlifecycle.go +++ b/examples/s3/setbucketlifecycle.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/lifecycle" ) @@ -36,7 +37,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/setbucketnotification.go b/examples/s3/setbucketnotification.go index a915fa4f9..519744efb 100644 --- a/examples/s3/setbucketnotification.go +++ b/examples/s3/setbucketnotification.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/notification" ) @@ -36,7 +37,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/setbucketpolicy.go b/examples/s3/setbucketpolicy.go index 8166a1e3b..9e47b29be 100644 --- a/examples/s3/setbucketpolicy.go +++ b/examples/s3/setbucketpolicy.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/setbucketreplication.go b/examples/s3/setbucketreplication.go index 3b668f77f..8e21079e8 100644 --- a/examples/s3/setbucketreplication.go +++ b/examples/s3/setbucketreplication.go @@ -25,6 +25,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/replication" ) @@ -37,7 +38,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/setbuckettagging.go b/examples/s3/setbuckettagging.go index 471f515cd..09f579d97 100644 --- a/examples/s3/setbuckettagging.go +++ b/examples/s3/setbuckettagging.go @@ -24,6 +24,7 @@ import ( "log" minio "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/tags" ) @@ -36,7 +37,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/setobjectlockconfig.go b/examples/s3/setobjectlockconfig.go index e2ead60f2..653c838b7 100644 --- a/examples/s3/setobjectlockconfig.go +++ b/examples/s3/setobjectlockconfig.go @@ -24,6 +24,7 @@ import ( "log" minio "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/statobject.go b/examples/s3/statobject.go index df2f1425e..667b9e4e6 100644 --- a/examples/s3/statobject.go +++ b/examples/s3/statobject.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } From af46bf94fbe89b32470d2976ba44cd77e50b3f4a Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sat, 18 Jul 2020 15:49:28 -0700 Subject: [PATCH 195/215] feat: Add ListenNotification API to listen for all events (#1343) --- README.md | 2 + api-bucket-notification.go | 23 +++++++---- docs/API.md | 50 +++++++++++++++++++++++ examples/minio/listen-notification.go | 58 +++++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 8 deletions(-) create mode 100644 examples/minio/listen-notification.go diff --git a/README.md b/README.md index b0d85237e..9aedbd0ed 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,7 @@ The full API Reference is available here. * [`GetBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#GetBucketNotification) * [`RemoveAllBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#RemoveAllBucketNotification) * [`ListenBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#ListenBucketNotification) (MinIO Extension) +* [`ListenNotification`](https://docs.min.io/docs/golang-client-api-reference#ListenNotification) (MinIO Extension) ### API Reference : File Object Operations * [`FPutObject`](https://docs.min.io/docs/golang-client-api-reference#FPutObject) @@ -212,6 +213,7 @@ The full API Reference is available here. * [getbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketnotification.go) * [removeallbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeallbucketnotification.go) * [listenbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/minio/listenbucketnotification.go) (MinIO Extension) +* [listennotification.go](https://github.com/minio/minio-go/blob/master/examples/minio/listen-notification.go) (MinIO Extension) ### Full Examples : File Object Operations * [fputobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/fputobject.go) diff --git a/api-bucket-notification.go b/api-bucket-notification.go index 8f198b116..1053cbae5 100644 --- a/api-bucket-notification.go +++ b/api-bucket-notification.go @@ -120,24 +120,31 @@ func processBucketNotificationResponse(bucketName string, resp *http.Response) ( return bucketNotification, nil } +// ListenNotification listen for all events, this is a MinIO specific API +func (c Client) ListenNotification(ctx context.Context, prefix, suffix string, events []string) <-chan notification.Info { + return c.ListenBucketNotification(ctx, "", prefix, suffix, events) +} + // ListenBucketNotification listen for bucket events, this is a MinIO specific API func (c Client) ListenBucketNotification(ctx context.Context, bucketName, prefix, suffix string, events []string) <-chan notification.Info { notificationInfoCh := make(chan notification.Info, 1) - const notificationCapacity = 1024 * 1024 + const notificationCapacity = 4 * 1024 * 1024 notificationEventBuffer := make([]byte, notificationCapacity) // Only success, start a routine to start reading line by line. go func(notificationInfoCh chan<- notification.Info) { defer close(notificationInfoCh) // Validate the bucket name. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - select { - case notificationInfoCh <- notification.Info{ - Err: err, - }: - case <-ctx.Done(): + if bucketName != "" { + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + select { + case notificationInfoCh <- notification.Info{ + Err: err, + }: + case <-ctx.Done(): + } + return } - return } // Check ARN partition to verify if listening bucket is supported diff --git a/docs/API.md b/docs/API.md index 5fdadb24d..7b1591056 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1497,6 +1497,56 @@ for notificationInfo := range minioClient.ListenBucketNotification(context.Backg } ``` + +### ListenNotification(context context.Context, prefix, suffix string, events []string) <-chan notification.Info +ListenNotification API receives bucket and object notification events through the notification channel. The returned notification channel has two fields 'Records' and 'Err'. + +- 'Records' holds the notifications received from the server. +- 'Err' indicates any error while processing the received notifications. + +NOTE: Notification channel is closed at the first occurrence of an error. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ | Bucket to listen notifications on | +|`prefix` | _string_ | Object key prefix to filter notifications for | +|`suffix` | _string_ | Object key suffix to filter notifications for | +|`events` | _[]string_ | Enables notifications for specific event types | + +__Return Values__ + +|Param |Type |Description | +|:---|:---| :---| +|`notificationInfo` | _chan notification.Info_ | Read channel for all notifications | + +__minio.NotificationInfo__ + +|Field |Type |Description | +|`notificationInfo.Records` | _[]notification.Event_ | Collection of notification events | +|`notificationInfo.Err` | _error_ | Carries any error occurred during the operation (Standard Error) | + +__Example__ + + +```go +// Listen for bucket notifications on "mybucket" filtered by prefix, suffix and events. +for notificationInfo := range minioClient.ListenNotification(context.Background(), "myprefix/", ".mysuffix", []string{ + "s3:BucketCreated:*", + "s3:BucketRemoved:*", + "s3:ObjectCreated:*", + "s3:ObjectAccessed:*", + "s3:ObjectRemoved:*", + }) { + if notificationInfo.Err != nil { + fmt.Println(notificationInfo.Err) + } + fmt.Println(notificationInfo) +} +``` + ### SetBucketLifecycle(ctx context.Context, bucketname, config *lifecycle.Configuration) error Set lifecycle on bucket or an object prefix. diff --git a/examples/minio/listen-notification.go b/examples/minio/listen-notification.go new file mode 100644 index 000000000..abbf84ac8 --- /dev/null +++ b/examples/minio/listen-notification.go @@ -0,0 +1,58 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "context" + "log" + + "github.com/minio/minio-go/v7" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + minioClient, err := minio.New("play.min.io", "YOUR-ACCESS", "YOUR-SECRET", true) + if err != nil { + log.Fatalln(err) + } + + // s3Client.TraceOn(os.Stderr) + + // Listen for bucket notifications on "mybucket" filtered by prefix, suffix and events. + for notificationInfo := range minioClient.ListenNotification(context.Background(), "PREFIX", "SUFFIX", []string{ + "s3:BucketCreated:*", + "s3:BucketRemoved:*", + "s3:ObjectCreated:*", + "s3:ObjectAccessed:*", + "s3:ObjectRemoved:*", + }) { + if notificationInfo.Err != nil { + log.Fatalln(notificationInfo.Err) + } + log.Println(notificationInfo) + } +} From f0e2f3ae36786e5e69f34fdaff106223ddba6082 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sat, 18 Jul 2020 16:57:21 -0700 Subject: [PATCH 196/215] fix: removing lifecycle properly (#1344) --- .github/workflows/go-windows.yml | 2 +- .github/workflows/go.yml | 2 +- api-bucket-lifecycle.go | 8 ++++---- api-bucket-notification.go | 4 ++-- api-bucket-policy.go | 6 +++--- api-bucket-replication.go | 2 +- api-bucket-versioning.go | 2 +- api-compose-object.go | 6 +++--- api-get-object-acl.go | 2 +- api-get-object.go | 2 +- api-list.go | 12 ++++++------ api-object-legal-hold.go | 4 ++-- api-object-lock.go | 4 ++-- api-object-retention.go | 4 ++-- api-object-tagging.go | 6 +++--- api-presigned.go | 6 +++--- api-put-bucket.go | 2 +- api-put-object-multipart.go | 6 +++--- api-put-object-streaming.go | 2 +- api-remove.go | 6 +++--- api-select.go | 2 +- api-stat.go | 4 ++-- api.go | 6 +++--- bucket-cache.go | 2 +- bucket-cache_test.go | 2 +- pkg/credentials/assume_role.go | 2 +- pkg/credentials/iam_aws.go | 6 +++--- pkg/credentials/sts_client_grants.go | 2 +- pkg/credentials/sts_ldap_identity.go | 2 +- pkg/credentials/sts_web_identity.go | 2 +- pkg/lifecycle/lifecycle.go | 8 ++++++++ pkg/signer/request-signature-streaming_test.go | 7 ++++--- pkg/signer/request-signature-v4_test.go | 2 +- pkg/signer/request-signature_test.go | 4 ++-- pkg/signer/test-utils_test.go | 4 ++-- 35 files changed, 76 insertions(+), 67 deletions(-) diff --git a/.github/workflows/go-windows.yml b/.github/workflows/go-windows.yml index 91d240691..7a592ce14 100644 --- a/.github/workflows/go-windows.yml +++ b/.github/workflows/go-windows.yml @@ -39,7 +39,7 @@ jobs: run: | New-Item -ItemType Directory -Path "$env:temp/certs-dir" Copy-Item -Path testcerts\* -Destination "$env:temp/certs-dir" - Invoke-WebRequest -Uri https://dl.minio.io/server/minio/release/windows-amd64/minio.exe -OutFile $HOME/minio.exe + Invoke-WebRequest -Uri https://dl.minio.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2020-07-14T19-14-30Z -OutFile $HOME/minio.exe Start-Process -NoNewWindow -FilePath "$HOME/minio.exe" -ArgumentList "-S", "$env:temp/certs-dir", "server", "$env:temp/fs{1...4}" $env:SSL_CERT_FILE = "$env:temp/certs-dir/public.crt" go run functional_tests.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index ccc2a235c..04d8ee408 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -39,7 +39,7 @@ jobs: SSL_CERT_FILE: /tmp/certs-dir/public.crt run: | sudo apt-get install devscripts - wget -O /tmp/minio https://dl.min.io/server/minio/release/linux-amd64/minio + wget -O /tmp/minio https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2020-07-14T19-14-30Z chmod +x /tmp/minio mkdir -p /tmp/certs-dir cp testcerts/* /tmp/certs-dir diff --git a/api-bucket-lifecycle.go b/api-bucket-lifecycle.go index daaceb52f..e1fac813c 100644 --- a/api-bucket-lifecycle.go +++ b/api-bucket-lifecycle.go @@ -37,7 +37,7 @@ func (c Client) SetBucketLifecycle(ctx context.Context, bucketName string, confi } // If lifecycle is empty then delete it. - if config == nil { + if config.Empty() { return c.removeBucketLifecycle(ctx, bucketName) } @@ -67,7 +67,7 @@ func (c Client) putBucketLifecycle(ctx context.Context, bucketName string, buf [ } // Execute PUT to upload a new bucket lifecycle. - resp, err := c.executeMethod(ctx, "PUT", reqMetadata) + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err @@ -88,7 +88,7 @@ func (c Client) removeBucketLifecycle(ctx context.Context, bucketName string) er urlValues.Set("lifecycle", "") // Execute DELETE on objectName. - resp, err := c.executeMethod(ctx, "DELETE", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, @@ -127,7 +127,7 @@ func (c Client) getBucketLifecycle(ctx context.Context, bucketName string) ([]by urlValues.Set("lifecycle", "") // Execute GET on bucket to get lifecycle. - resp, err := c.executeMethod(ctx, "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, }) diff --git a/api-bucket-notification.go b/api-bucket-notification.go index 1053cbae5..76787ecab 100644 --- a/api-bucket-notification.go +++ b/api-bucket-notification.go @@ -59,7 +59,7 @@ func (c Client) SetBucketNotification(ctx context.Context, bucketName string, co } // Execute PUT to upload a new bucket notification. - resp, err := c.executeMethod(ctx, "PUT", reqMetadata) + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err @@ -92,7 +92,7 @@ func (c Client) getBucketNotification(ctx context.Context, bucketName string) (n urlValues.Set("notification", "") // Execute GET on bucket to list objects. - resp, err := c.executeMethod(ctx, "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, diff --git a/api-bucket-policy.go b/api-bucket-policy.go index 8b3a20034..72676f344 100644 --- a/api-bucket-policy.go +++ b/api-bucket-policy.go @@ -57,7 +57,7 @@ func (c Client) putBucketPolicy(ctx context.Context, bucketName, policy string) } // Execute PUT to upload a new bucket policy. - resp, err := c.executeMethod(ctx, "PUT", reqMetadata) + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err @@ -78,7 +78,7 @@ func (c Client) removeBucketPolicy(ctx context.Context, bucketName string) error urlValues.Set("policy", "") // Execute DELETE on objectName. - resp, err := c.executeMethod(ctx, "DELETE", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, @@ -115,7 +115,7 @@ func (c Client) getBucketPolicy(ctx context.Context, bucketName string) (string, urlValues.Set("policy", "") // Execute GET on bucket to list objects. - resp, err := c.executeMethod(ctx, "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, diff --git a/api-bucket-replication.go b/api-bucket-replication.go index 116a17ee1..bfd5ea436 100644 --- a/api-bucket-replication.go +++ b/api-bucket-replication.go @@ -127,7 +127,7 @@ func (c Client) getBucketReplication(ctx context.Context, bucketName string) (cf urlValues.Set("replication", "") // Execute GET on bucket to get replication config. - resp, err := c.executeMethod(ctx, "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, }) diff --git a/api-bucket-versioning.go b/api-bucket-versioning.go index c521e4870..0889d43b0 100644 --- a/api-bucket-versioning.go +++ b/api-bucket-versioning.go @@ -53,7 +53,7 @@ func (c Client) SetBucketVersioning(ctx context.Context, bucketName string, conf } // Execute PUT to set a bucket versioning. - resp, err := c.executeMethod(ctx, "PUT", reqMetadata) + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err diff --git a/api-compose-object.go b/api-compose-object.go index 421850a1d..cb2e899e5 100644 --- a/api-compose-object.go +++ b/api-compose-object.go @@ -215,7 +215,7 @@ func (c Client) copyObjectDo(ctx context.Context, srcBucket, srcObject, destBuck headers.Set("x-amz-copy-source", s3utils.EncodePath(srcBucket+"/"+srcObject)) // Send upload-part-copy request - resp, err := c.executeMethod(ctx, "PUT", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodPut, requestMetadata{ bucketName: destBucket, objectName: destObject, customHeader: headers, @@ -268,7 +268,7 @@ func (c Client) copyObjectPartDo(ctx context.Context, srcBucket, srcObject, dest queryValues.Set("partNumber", strconv.Itoa(partID)) queryValues.Set("uploadId", uploadID) - resp, err := c.executeMethod(ctx, "PUT", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodPut, requestMetadata{ bucketName: destBucket, objectName: destObject, customHeader: headers, @@ -306,7 +306,7 @@ func (c Client) uploadPartCopy(ctx context.Context, bucket, object, uploadID str urlValues.Set("uploadId", uploadID) // Send upload-part-copy request - resp, err := c.executeMethod(ctx, "PUT", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodPut, requestMetadata{ bucketName: bucket, objectName: object, customHeader: headers, diff --git a/api-get-object-acl.go b/api-get-object-acl.go index 86a30bb73..afa53079d 100644 --- a/api-get-object-acl.go +++ b/api-get-object-acl.go @@ -42,7 +42,7 @@ type accessControlPolicy struct { // GetObjectACL get object ACLs func (c Client) GetObjectACL(ctx context.Context, bucketName, objectName string) (*ObjectInfo, error) { - resp, err := c.executeMethod(ctx, "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: url.Values{ diff --git a/api-get-object.go b/api-get-object.go index 78147a97a..ac9bbdbe0 100644 --- a/api-get-object.go +++ b/api-get-object.go @@ -619,7 +619,7 @@ func (c Client) getObject(ctx context.Context, bucketName, objectName string, op } // Execute GET on objectName. - resp, err := c.executeMethod(ctx, "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, diff --git a/api-list.go b/api-list.go index 784542416..7996c11e9 100644 --- a/api-list.go +++ b/api-list.go @@ -38,7 +38,7 @@ import ( // func (c Client) ListBuckets(ctx context.Context) ([]BucketInfo, error) { // Execute GET on service. - resp, err := c.executeMethod(ctx, "GET", requestMetadata{contentSHA256Hex: emptySHA256Hex}) + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{contentSHA256Hex: emptySHA256Hex}) defer closeResponse(resp) if err != nil { return nil, err @@ -198,7 +198,7 @@ func (c Client) listObjectsV2Query(ctx context.Context, bucketName, objectPrefix } // Execute GET on bucket to list objects. - resp, err := c.executeMethod(ctx, "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, @@ -479,7 +479,7 @@ func (c Client) listObjectVersionsQuery(ctx context.Context, bucketName, prefix, urlValues.Set("encoding-type", "url") // Execute GET on bucket to list objects. - resp, err := c.executeMethod(ctx, "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, @@ -567,7 +567,7 @@ func (c Client) listObjectsQuery(ctx context.Context, bucketName, objectPrefix, urlValues.Set("encoding-type", "url") // Execute GET on bucket to list objects. - resp, err := c.executeMethod(ctx, "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, @@ -795,7 +795,7 @@ func (c Client) listMultipartUploadsQuery(ctx context.Context, bucketName, keyMa } // Execute GET on bucketName to list multipart uploads. - resp, err := c.executeMethod(ctx, "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, @@ -915,7 +915,7 @@ func (c Client) listObjectPartsQuery(ctx context.Context, bucketName, objectName } // Execute GET on objectName to get list of parts. - resp, err := c.executeMethod(ctx, "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, diff --git a/api-object-legal-hold.go b/api-object-legal-hold.go index 8dbf4b04c..b139c1687 100644 --- a/api-object-legal-hold.go +++ b/api-object-legal-hold.go @@ -121,7 +121,7 @@ func (c Client) PutObjectLegalHold(ctx context.Context, bucketName, objectName s } // Execute PUT Object Legal Hold. - resp, err := c.executeMethod(ctx, "PUT", reqMetadata) + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err @@ -152,7 +152,7 @@ func (c Client) GetObjectLegalHold(ctx context.Context, bucketName, objectName s } // Execute GET on bucket to list objects. - resp, err := c.executeMethod(ctx, "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, diff --git a/api-object-lock.go b/api-object-lock.go index 4b5f72dc2..29f52b054 100644 --- a/api-object-lock.go +++ b/api-object-lock.go @@ -170,7 +170,7 @@ func (c Client) SetBucketObjectLockConfig(ctx context.Context, bucketName string } // Execute PUT bucket object lock configuration. - resp, err := c.executeMethod(ctx, "PUT", reqMetadata) + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err @@ -194,7 +194,7 @@ func (c Client) GetObjectLockConfig(ctx context.Context, bucketName string) (obj urlValues.Set("object-lock", "") // Execute GET on bucket to list objects. - resp, err := c.executeMethod(ctx, "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, diff --git a/api-object-retention.go b/api-object-retention.go index 0b85b6d27..54f2762de 100644 --- a/api-object-retention.go +++ b/api-object-retention.go @@ -112,7 +112,7 @@ func (c Client) PutObjectRetention(ctx context.Context, bucketName, objectName s } // Execute PUT Object Retention. - resp, err := c.executeMethod(ctx, "PUT", reqMetadata) + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err @@ -141,7 +141,7 @@ func (c Client) GetObjectRetention(ctx context.Context, bucketName, objectName, urlValues.Set("versionId", versionID) } // Execute GET on bucket to list objects. - resp, err := c.executeMethod(ctx, "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, diff --git a/api-object-tagging.go b/api-object-tagging.go index ad0775870..2709efcd1 100644 --- a/api-object-tagging.go +++ b/api-object-tagging.go @@ -66,7 +66,7 @@ func (c Client) PutObjectTagging(ctx context.Context, bucketName, objectName str } // Execute PUT to set a object tagging. - resp, err := c.executeMethod(ctx, "PUT", reqMetadata) + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err @@ -98,7 +98,7 @@ func (c Client) GetObjectTagging(ctx context.Context, bucketName, objectName str } // Execute GET on object to get object tag(s) - resp, err := c.executeMethod(ctx, "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, @@ -136,7 +136,7 @@ func (c Client) RemoveObjectTagging(ctx context.Context, bucketName, objectName } // Execute DELETE on object to remove object tag(s) - resp, err := c.executeMethod(ctx, "DELETE", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, diff --git a/api-presigned.go b/api-presigned.go index c06a4fe7d..80c363da5 100644 --- a/api-presigned.go +++ b/api-presigned.go @@ -69,7 +69,7 @@ func (c Client) PresignedGetObject(ctx context.Context, bucketName string, objec if err = s3utils.CheckValidObjectName(objectName); err != nil { return nil, err } - return c.presignURL(ctx, "GET", bucketName, objectName, expires, reqParams) + return c.presignURL(ctx, http.MethodGet, bucketName, objectName, expires, reqParams) } // PresignedHeadObject - Returns a presigned URL to access @@ -80,7 +80,7 @@ func (c Client) PresignedHeadObject(ctx context.Context, bucketName string, obje if err = s3utils.CheckValidObjectName(objectName); err != nil { return nil, err } - return c.presignURL(ctx, "HEAD", bucketName, objectName, expires, reqParams) + return c.presignURL(ctx, http.MethodHead, bucketName, objectName, expires, reqParams) } // PresignedPutObject - Returns a presigned URL to upload an object @@ -90,7 +90,7 @@ func (c Client) PresignedPutObject(ctx context.Context, bucketName string, objec if err = s3utils.CheckValidObjectName(objectName); err != nil { return nil, err } - return c.presignURL(ctx, "PUT", bucketName, objectName, expires, nil) + return c.presignURL(ctx, http.MethodPut, bucketName, objectName, expires, nil) } // Presign - returns a presigned URL for any http method of your choice diff --git a/api-put-bucket.go b/api-put-bucket.go index d8b644328..df9fe98af 100644 --- a/api-put-bucket.go +++ b/api-put-bucket.go @@ -87,7 +87,7 @@ func (c Client) doMakeBucket(ctx context.Context, bucketName string, location st } // Execute PUT to create a new bucket. - resp, err := c.executeMethod(ctx, "PUT", reqMetadata) + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err diff --git a/api-put-object-multipart.go b/api-put-object-multipart.go index 5d8da15a8..25e633fcf 100644 --- a/api-put-object-multipart.go +++ b/api-put-object-multipart.go @@ -208,7 +208,7 @@ func (c Client) initiateMultipartUpload(ctx context.Context, bucketName, objectN } // Execute POST on an objectName to initiate multipart upload. - resp, err := c.executeMethod(ctx, "POST", reqMetadata) + resp, err := c.executeMethod(ctx, http.MethodPost, reqMetadata) defer closeResponse(resp) if err != nil { return initiateMultipartUploadResult{}, err @@ -279,7 +279,7 @@ func (c Client) uploadPart(ctx context.Context, bucketName, objectName, uploadID } // Execute PUT on each part. - resp, err := c.executeMethod(ctx, "PUT", reqMetadata) + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return ObjectPart{}, err @@ -330,7 +330,7 @@ func (c Client) completeMultipartUpload(ctx context.Context, bucketName, objectN } // Execute POST to complete multipart upload for an objectName. - resp, err := c.executeMethod(ctx, "POST", reqMetadata) + resp, err := c.executeMethod(ctx, http.MethodPost, reqMetadata) defer closeResponse(resp) if err != nil { return UploadInfo{}, err diff --git a/api-put-object-streaming.go b/api-put-object-streaming.go index 98c289d7b..144dbb67e 100644 --- a/api-put-object-streaming.go +++ b/api-put-object-streaming.go @@ -443,7 +443,7 @@ func (c Client) putObjectDo(ctx context.Context, bucketName, objectName string, reqMetadata.queryValues = urlValues } // Execute PUT an objectName. - resp, err := c.executeMethod(ctx, "PUT", reqMetadata) + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return UploadInfo{}, err diff --git a/api-remove.go b/api-remove.go index 681b3fc55..6c2ab7802 100644 --- a/api-remove.go +++ b/api-remove.go @@ -38,7 +38,7 @@ func (c Client) RemoveBucket(ctx context.Context, bucketName string) error { return err } // Execute DELETE on bucket. - resp, err := c.executeMethod(ctx, "DELETE", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ bucketName: bucketName, contentSHA256Hex: emptySHA256Hex, }) @@ -90,7 +90,7 @@ func (c Client) RemoveObject(ctx context.Context, bucketName, objectName string, headers.Set(amzBypassGovernance, "true") } // Execute DELETE on objectName. - resp, err := c.executeMethod(ctx, "DELETE", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ bucketName: bucketName, objectName: objectName, contentSHA256Hex: emptySHA256Hex, @@ -308,7 +308,7 @@ func (c Client) abortMultipartUpload(ctx context.Context, bucketName, objectName urlValues.Set("uploadId", uploadID) // Execute DELETE on multipart upload. - resp, err := c.executeMethod(ctx, "DELETE", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, diff --git a/api-select.go b/api-select.go index be74fa353..340fa5299 100644 --- a/api-select.go +++ b/api-select.go @@ -450,7 +450,7 @@ func (c Client) SelectObjectContent(ctx context.Context, bucketName, objectName urlValues.Set("select-type", "2") // Execute POST on bucket/object. - resp, err := c.executeMethod(ctx, "POST", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodPost, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, diff --git a/api-stat.go b/api-stat.go index 0cdac7e71..ea9c30970 100644 --- a/api-stat.go +++ b/api-stat.go @@ -34,7 +34,7 @@ func (c Client) BucketExists(ctx context.Context, bucketName string) (bool, erro } // Execute HEAD on bucketName. - resp, err := c.executeMethod(ctx, "HEAD", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodHead, requestMetadata{ bucketName: bucketName, contentSHA256Hex: emptySHA256Hex, }) @@ -85,7 +85,7 @@ func (c Client) statObject(ctx context.Context, bucketName, objectName string, o } // Execute HEAD on objectName. - resp, err := c.executeMethod(ctx, "HEAD", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodHead, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, diff --git a/api.go b/api.go index cd14a5fda..f3d9929fc 100644 --- a/api.go +++ b/api.go @@ -679,7 +679,7 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque func (c Client) newRequest(ctx context.Context, method string, metadata requestMetadata) (req *http.Request, err error) { // If no method is supplied default to 'POST'. if method == "" { - method = "POST" + method = http.MethodPost } location := metadata.bucketLocation @@ -699,7 +699,7 @@ func (c Client) newRequest(ctx context.Context, method string, metadata requestM // Look if target url supports virtual host. // We explicitly disallow MakeBucket calls to not use virtual DNS style, // since the resolution may fail. - isMakeBucket := (metadata.objectName == "" && method == "PUT" && len(metadata.queryValues) == 0) + isMakeBucket := (metadata.objectName == "" && method == http.MethodPut && len(metadata.queryValues) == 0) isVirtualHost := c.isVirtualHostStyleRequest(*c.endpointURL, metadata.bucketName) && !isMakeBucket // Construct a new target URL. @@ -793,7 +793,7 @@ func (c Client) newRequest(ctx context.Context, method string, metadata requestM case signerType.IsV2(): // Add signature version '2' authorization header. req = signer.SignV2(*req, accessKeyID, secretAccessKey, isVirtualHost) - case metadata.objectName != "" && metadata.queryValues == nil && method == "PUT" && metadata.customHeader.Get("X-Amz-Copy-Source") == "" && !c.secure: + case metadata.objectName != "" && metadata.queryValues == nil && method == http.MethodPut && metadata.customHeader.Get("X-Amz-Copy-Source") == "" && !c.secure: // Streaming signature is used by default for a PUT object request. Additionally we also // look if the initialized client is secure, if yes then we don't need to perform // streaming signature. diff --git a/bucket-cache.go b/bucket-cache.go index d60994424..7d485a6b1 100644 --- a/bucket-cache.go +++ b/bucket-cache.go @@ -198,7 +198,7 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro } // Get a new HTTP request for the method. - req, err := http.NewRequest("GET", urlStr, nil) + req, err := http.NewRequest(http.MethodGet, urlStr, nil) if err != nil { return nil, err } diff --git a/bucket-cache_test.go b/bucket-cache_test.go index d70a08515..b95857b36 100644 --- a/bucket-cache_test.go +++ b/bucket-cache_test.go @@ -78,7 +78,7 @@ func TestGetBucketLocationRequest(t *testing.T) { targetURL.RawQuery = urlValues.Encode() // Get a new HTTP request for the method. - req, err := http.NewRequest("GET", targetURL.String(), nil) + req, err := http.NewRequest(http.MethodGet, targetURL.String(), nil) if err != nil { return nil, err } diff --git a/pkg/credentials/assume_role.go b/pkg/credentials/assume_role.go index 7456211e7..cc88a9ab6 100644 --- a/pkg/credentials/assume_role.go +++ b/pkg/credentials/assume_role.go @@ -170,7 +170,7 @@ func getAssumeRoleCredentials(clnt *http.Client, endpoint string, opts STSAssume } postBody.Seek(0, 0) - req, err := http.NewRequest("POST", u.String(), postBody) + req, err := http.NewRequest(http.MethodPost, u.String(), postBody) if err != nil { return AssumeRoleResponse{}, err } diff --git a/pkg/credentials/iam_aws.go b/pkg/credentials/iam_aws.go index 25135e2c9..ceeab84dd 100644 --- a/pkg/credentials/iam_aws.go +++ b/pkg/credentials/iam_aws.go @@ -190,7 +190,7 @@ func getIAMRoleURL(endpoint string) (*url.URL, error) { // or there is an error making or receiving the request. // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html func listRoleNames(client *http.Client, u *url.URL) ([]string, error) { - req, err := http.NewRequest("GET", u.String(), nil) + req, err := http.NewRequest(http.MethodGet, u.String(), nil) if err != nil { return nil, err } @@ -217,7 +217,7 @@ func listRoleNames(client *http.Client, u *url.URL) ([]string, error) { } func getEcsTaskCredentials(client *http.Client, endpoint string) (ec2RoleCredRespBody, error) { - req, err := http.NewRequest("GET", endpoint, nil) + req, err := http.NewRequest(http.MethodGet, endpoint, nil) if err != nil { return ec2RoleCredRespBody{}, err } @@ -273,7 +273,7 @@ func getCredentials(client *http.Client, endpoint string) (ec2RoleCredRespBody, // $ curl http://169.254.169.254/latest/meta-data/iam/security-credentials/s3access // u.Path = path.Join(u.Path, roleName) - req, err := http.NewRequest("GET", u.String(), nil) + req, err := http.NewRequest(http.MethodGet, u.String(), nil) if err != nil { return ec2RoleCredRespBody{}, err } diff --git a/pkg/credentials/sts_client_grants.go b/pkg/credentials/sts_client_grants.go index 03134c3d2..e89d5d4d8 100644 --- a/pkg/credentials/sts_client_grants.go +++ b/pkg/credentials/sts_client_grants.go @@ -122,7 +122,7 @@ func getClientGrantsCredentials(clnt *http.Client, endpoint string, } u.RawQuery = v.Encode() - req, err := http.NewRequest("POST", u.String(), nil) + req, err := http.NewRequest(http.MethodPost, u.String(), nil) if err != nil { return AssumeRoleWithClientGrantsResponse{}, err } diff --git a/pkg/credentials/sts_ldap_identity.go b/pkg/credentials/sts_ldap_identity.go index b72ac061c..abbf61641 100644 --- a/pkg/credentials/sts_ldap_identity.go +++ b/pkg/credentials/sts_ldap_identity.go @@ -85,7 +85,7 @@ func (k *LDAPIdentity) Retrieve() (value Value, err error) { u.RawQuery = v.Encode() - req, kerr := http.NewRequest("POST", u.String(), nil) + req, kerr := http.NewRequest(http.MethodPost, u.String(), nil) if kerr != nil { err = kerr return diff --git a/pkg/credentials/sts_web_identity.go b/pkg/credentials/sts_web_identity.go index a197161a3..5a5f6405e 100644 --- a/pkg/credentials/sts_web_identity.go +++ b/pkg/credentials/sts_web_identity.go @@ -133,7 +133,7 @@ func getWebIdentityCredentials(clnt *http.Client, endpoint, roleARN, roleSession u.RawQuery = v.Encode() - req, err := http.NewRequest("POST", u.String(), nil) + req, err := http.NewRequest(http.MethodPost, u.String(), nil) if err != nil { return AssumeRoleWithWebIdentityResponse{}, err } diff --git a/pkg/lifecycle/lifecycle.go b/pkg/lifecycle/lifecycle.go index 0a6f822af..4dfa110e6 100644 --- a/pkg/lifecycle/lifecycle.go +++ b/pkg/lifecycle/lifecycle.go @@ -266,6 +266,14 @@ type Configuration struct { Rules []Rule `xml:"Rule"` } +// Empty check if lifecycle configuration is empty +func (c *Configuration) Empty() bool { + if c == nil { + return true + } + return len(c.Rules) == 0 +} + // NewConfiguration initializes a fresh lifecycle configuration // for manipulation, such as setting and removing lifecycle rules // and filters. diff --git a/pkg/signer/request-signature-streaming_test.go b/pkg/signer/request-signature-streaming_test.go index 4ee08b4a0..43b614ee8 100644 --- a/pkg/signer/request-signature-streaming_test.go +++ b/pkg/signer/request-signature-streaming_test.go @@ -20,6 +20,7 @@ package signer import ( "bytes" "io/ioutil" + "net/http" "testing" "time" ) @@ -31,7 +32,7 @@ func TestGetSeedSignature(t *testing.T) { data := bytes.Repeat([]byte("a"), dataLen) body := ioutil.NopCloser(bytes.NewReader(data)) - req := NewRequest("PUT", "/examplebucket/chunkObject.txt", body) + req := NewRequest(http.MethodPut, "/examplebucket/chunkObject.txt", body) req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") req.Host = "s3.amazonaws.com" @@ -67,7 +68,7 @@ func TestSetStreamingAuthorization(t *testing.T) { secretAccessKeyID := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" accessKeyID := "AKIAIOSFODNN7EXAMPLE" - req := NewRequest("PUT", "/examplebucket/chunkObject.txt", nil) + req := NewRequest(http.MethodPut, "/examplebucket/chunkObject.txt", nil) req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") req.Host = "" req.URL.Host = "s3.amazonaws.com" @@ -91,7 +92,7 @@ func TestStreamingReader(t *testing.T) { accessKeyID := "AKIAIOSFODNN7EXAMPLE" dataLen := int64(65 * 1024) - req := NewRequest("PUT", "/examplebucket/chunkObject.txt", nil) + req := NewRequest(http.MethodPut, "/examplebucket/chunkObject.txt", nil) req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") req.ContentLength = 65 * 1024 req.Host = "" diff --git a/pkg/signer/request-signature-v4_test.go b/pkg/signer/request-signature-v4_test.go index 96e33286f..c67a47230 100644 --- a/pkg/signer/request-signature-v4_test.go +++ b/pkg/signer/request-signature-v4_test.go @@ -38,7 +38,7 @@ func TestRequestHost(t *testing.T) { func buildRequest(serviceName, region, body string) (*http.Request, io.ReadSeeker) { endpoint := "https://" + serviceName + "." + region + ".amazonaws.com" reader := strings.NewReader(body) - req, _ := http.NewRequest("POST", endpoint, reader) + req, _ := http.NewRequest(http.MethodPost, endpoint, reader) req.URL.Opaque = "//example.org/bucket/key-._~,!@#$%^&*()" req.Header.Add("X-Amz-Target", "prefix.Operation") req.Header.Add("Content-Type", "application/x-amz-json-1.0") diff --git a/pkg/signer/request-signature_test.go b/pkg/signer/request-signature_test.go index cc14bc4c9..3709160a3 100644 --- a/pkg/signer/request-signature_test.go +++ b/pkg/signer/request-signature_test.go @@ -25,7 +25,7 @@ import ( // Tests signature calculation. func TestSignatureCalculationV4(t *testing.T) { - req, err := http.NewRequest("GET", "https://s3.amazonaws.com", nil) + req, err := http.NewRequest(http.MethodGet, "https://s3.amazonaws.com", nil) if err != nil { t.Fatal("Error:", err) } @@ -61,7 +61,7 @@ func TestSignatureCalculationV2(t *testing.T) { } for i, testCase := range testCases { - req, err := http.NewRequest("GET", testCase.endpointURL, nil) + req, err := http.NewRequest(http.MethodGet, testCase.endpointURL, nil) if err != nil { t.Fatalf("Test %d, Error: %v", i+1, err) } diff --git a/pkg/signer/test-utils_test.go b/pkg/signer/test-utils_test.go index 56ba5727b..c39ab83bc 100644 --- a/pkg/signer/test-utils_test.go +++ b/pkg/signer/test-utils_test.go @@ -43,7 +43,7 @@ import ( // // The Request.Proto is always HTTP/1.1. // -// An empty method means "GET". +// An empty method means http.MethodGet. // // The provided body may be nil. If the body is of type *bytes.Reader, // *strings.Reader, or *bytes.Buffer, the Request.ContentLength is @@ -53,7 +53,7 @@ import ( // panic is acceptable. func NewRequest(method, target string, body io.Reader) *http.Request { if method == "" { - method = "GET" + method = http.MethodGet } req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(method + " " + target + " HTTP/1.0\r\n\r\n"))) if err != nil { From af9c411483053e4475683ff15b030b38f54cedd7 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sat, 18 Jul 2020 21:16:30 -0700 Subject: [PATCH 197/215] update all remaining docs --- README.md | 4 +- docs/API.md | 59 +++++++++++++++++---------- examples/minio/listen-notification.go | 6 ++- examples/s3/get-encrypted-object.go | 6 ++- examples/s3/getobject.go | 6 ++- examples/s3/getobjectacl.go | 9 ++-- examples/s3/getobjectlegalhold.go | 6 ++- examples/s3/getobjectlockconfig.go | 6 ++- examples/s3/getobjectretention.go | 6 ++- examples/s3/putobjectlegalhold.go | 6 ++- examples/s3/putobjectretention.go | 6 ++- examples/s3/selectobject.go | 6 ++- 12 files changed, 89 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 9aedbd0ed..7c2b6d78c 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,7 @@ MinIO client requires the following four parameters specified to connect to an A | Parameter | Description| | :--- | :--- | | endpoint | URL to object storage service. | -| accessKeyID | Access key is the user ID that uniquely identifies your account. | -| secretAccessKey | Secret key is the password to your account. | -| secure | Set this value to 'true' to enable secure (HTTPS) access. | +| _minio.Options_ | All the options such as credentials, custom transport etc. | ```go diff --git a/docs/API.md b/docs/API.md index 7b1591056..2ac72c8f2 100644 --- a/docs/API.md +++ b/docs/API.md @@ -8,21 +8,28 @@ package main import ( - "fmt" + "log" - "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { - // Use a secure connection. - ssl := true + endpoint := "play.min.io" + accessKeyID := "Q3AM3UQ867SPQQA43P2F" + secretAccessKey := "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" + useSSL := true - // Initialize minio client object. - minioClient, err := minio.New("play.min.io", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", ssl) - if err != nil { - fmt.Println(err) - return - } + // Initialize minio client object. + minioClient, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: useSSL, + }) + if err != nil { + log.Fatalln(err) + } + + log.Printf("%#v\n", minioClient) // minioClient is now setup } ``` @@ -35,14 +42,15 @@ import ( "fmt" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/credentials" ) func main() { - // Use a secure connection. - ssl := true - // Initialize minio client object. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ssl) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { fmt.Println(err) return @@ -1010,10 +1018,10 @@ __Return Values__ ```go // Initialize minio client object. - minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL) - if err != nil { - log.Fatalln(err) - } + minioClient, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: useSSL, + }) opts := minio.SelectObjectOptions{ Expression: "select count(*) from s3object", @@ -1634,7 +1642,10 @@ __Return Values__ __Example__ ```go -s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) +s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, +}) if err != nil { log.Fatalln(err) } @@ -1669,7 +1680,10 @@ __Return Values__ __Example__ ```go -s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) +s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, +}) if err != nil { log.Fatalln(err) } @@ -1866,7 +1880,10 @@ __Return Values__ __Example__ ```go -s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) +s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, +}) if err != nil { log.Fatalln(err) } diff --git a/examples/minio/listen-notification.go b/examples/minio/listen-notification.go index abbf84ac8..6a9c54e17 100644 --- a/examples/minio/listen-notification.go +++ b/examples/minio/listen-notification.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - minioClient, err := minio.New("play.min.io", "YOUR-ACCESS", "YOUR-SECRET", true) + minioClient, err := minio.New("play.min.io", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/get-encrypted-object.go b/examples/s3/get-encrypted-object.go index 732c7ad9b..fdfe5525e 100644 --- a/examples/s3/get-encrypted-object.go +++ b/examples/s3/get-encrypted-object.go @@ -26,6 +26,7 @@ import ( "os" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/encrypt" ) @@ -38,7 +39,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getobject.go b/examples/s3/getobject.go index 0bdd4ab87..7fec93f12 100644 --- a/examples/s3/getobject.go +++ b/examples/s3/getobject.go @@ -26,6 +26,7 @@ import ( "os" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -37,7 +38,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getobjectacl.go b/examples/s3/getobjectacl.go index 9262c0942..0581ce1e8 100644 --- a/examples/s3/getobjectacl.go +++ b/examples/s3/getobjectacl.go @@ -25,6 +25,7 @@ import ( "log" minio "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -36,10 +37,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) - if err != nil { - log.Fatalln(err) - } + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) objectInfo, err := s3Client.GetObjectACL(context.Background(), "my-bucketname", "my-objectname") if err != nil { diff --git a/examples/s3/getobjectlegalhold.go b/examples/s3/getobjectlegalhold.go index 41d763865..018b96f9e 100644 --- a/examples/s3/getobjectlegalhold.go +++ b/examples/s3/getobjectlegalhold.go @@ -25,6 +25,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -36,7 +37,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getobjectlockconfig.go b/examples/s3/getobjectlockconfig.go index 547a007a9..f00ec0ec7 100644 --- a/examples/s3/getobjectlockconfig.go +++ b/examples/s3/getobjectlockconfig.go @@ -25,6 +25,7 @@ import ( "log" minio "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -36,7 +37,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getobjectretention.go b/examples/s3/getobjectretention.go index 43c6f351b..8f19c713d 100644 --- a/examples/s3/getobjectretention.go +++ b/examples/s3/getobjectretention.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/putobjectlegalhold.go b/examples/s3/putobjectlegalhold.go index 9d9842fed..453628a06 100644 --- a/examples/s3/putobjectlegalhold.go +++ b/examples/s3/putobjectlegalhold.go @@ -24,6 +24,7 @@ import ( "log" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -35,7 +36,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/putobjectretention.go b/examples/s3/putobjectretention.go index 3b2be844e..a519f44a2 100644 --- a/examples/s3/putobjectretention.go +++ b/examples/s3/putobjectretention.go @@ -25,6 +25,7 @@ import ( "time" "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -36,7 +37,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/selectobject.go b/examples/s3/selectobject.go index 249f34dd4..ea06aff3c 100644 --- a/examples/s3/selectobject.go +++ b/examples/s3/selectobject.go @@ -26,6 +26,7 @@ import ( "os" minio "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -37,7 +38,10 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ + Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), + Secure: true, + }) if err != nil { log.Fatalln(err) } From f70f59e68c14bf04213e02ba88325268f74246df Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sun, 19 Jul 2020 10:54:15 -0700 Subject: [PATCH 198/215] allow KMS tests to be run in the CI/CD (#1345) --- .github/workflows/go-windows.yml | 1 + .github/workflows/go.yml | 1 + functional_tests.go | 36 +++++++++++++++----------------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/go-windows.yml b/.github/workflows/go-windows.yml index 7a592ce14..0e4909c9d 100644 --- a/.github/workflows/go-windows.yml +++ b/.github/workflows/go-windows.yml @@ -33,6 +33,7 @@ jobs: ACCESS_KEY: minio SECRET_KEY: minio123 ENABLE_HTTPS: 1 + MINIO_KMS_MASTER_KEY: my-minio-key:6368616e676520746869732070617373776f726420746f206120736563726574 MINIO_ACCESS_KEY: minio MINIO_SECRET_KEY: minio123 GO111MODULE: on diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 04d8ee408..d54fbf021 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -35,6 +35,7 @@ jobs: ENABLE_HTTPS: 1 MINIO_ACCESS_KEY: minio MINIO_SECRET_KEY: minio123 + MINIO_KMS_MASTER_KEY: my-minio-key:6368616e676520746869732070617373776f726420746f206120736563726574 GO111MODULE: on SSL_CERT_FILE: /tmp/certs-dir/public.crt run: | diff --git a/functional_tests.go b/functional_tests.go index a4502c02b..c7bc86d52 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -61,7 +61,6 @@ const ( accessKey = "ACCESS_KEY" secretKey = "SECRET_KEY" enableHTTPS = "ENABLE_HTTPS" - enableKMS = "ENABLE_KMS" ) type mintJSONFormatter struct { @@ -11188,7 +11187,6 @@ func main() { log.SetLevel(log.InfoLevel) tls := mustParseBool(os.Getenv(enableHTTPS)) - kmsEnabled := mustParseBool(os.Getenv(enableKMS)) // execute tests if isFullMode() { testMakeBucketErrorV2() @@ -11269,23 +11267,23 @@ func main() { testSSECEncryptedToUnencryptedCopyPart() testUnencryptedToSSECCopyObjectPart() testUnencryptedToUnencryptedCopyPart() - if kmsEnabled { - testSSES3EncryptionPutGet() - testSSES3EncryptionFPut() - testSSES3EncryptedGetObjectReadAtFunctional() - testSSES3EncryptedGetObjectReadSeekFunctional() - testEncryptedSSECToSSES3CopyObject() - testEncryptedSSES3ToSSECCopyObject() - testEncryptedSSES3ToSSES3CopyObject() - testEncryptedSSES3ToUnencryptedCopyObject() - testUnencryptedToSSES3CopyObject() - testSSECEncryptedToSSES3CopyObjectPart() - testUnencryptedToSSES3CopyObjectPart() - testSSES3EncryptedToSSECCopyObjectPart() - testSSES3EncryptedToUnencryptedCopyPart() - testSSES3EncryptedToSSES3CopyObjectPart() - } - } + testEncryptedSSECToSSES3CopyObject() + testEncryptedSSES3ToSSECCopyObject() + testSSECEncryptedToSSES3CopyObjectPart() + testSSES3EncryptedToSSECCopyObjectPart() + } + + // KMS tests + testSSES3EncryptionPutGet() + testSSES3EncryptionFPut() + testSSES3EncryptedGetObjectReadAtFunctional() + testSSES3EncryptedGetObjectReadSeekFunctional() + testEncryptedSSES3ToSSES3CopyObject() + testEncryptedSSES3ToUnencryptedCopyObject() + testUnencryptedToSSES3CopyObject() + testUnencryptedToSSES3CopyObjectPart() + testSSES3EncryptedToUnencryptedCopyPart() + testSSES3EncryptedToSSES3CopyObjectPart() } else { testFunctional() testFunctionalV2() From d977bd441dbb6df8e9c202e6bea33195aefb0d6c Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 21 Jul 2020 08:14:44 -0700 Subject: [PATCH 199/215] display appropriate funcName with nested callers (#1348) --- functional_tests.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/functional_tests.go b/functional_tests.go index c7bc86d52..055735b7f 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -309,7 +309,11 @@ func isFullMode() bool { } func getFuncName() string { - pc, _, _, _ := runtime.Caller(1) + return getFuncNameLoc(2) +} + +func getFuncNameLoc(caller int) string { + pc, _, _, _ := runtime.Caller(caller) return strings.TrimPrefix(runtime.FuncForPC(pc).Name(), "main.") } @@ -7054,7 +7058,7 @@ func testEncryptedEmptyObject() { func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, sseDst encrypt.ServerSide) { // initialize logging params startTime := time.Now() - testName := getFuncName() + testName := getFuncNameLoc(2) function := "CopyObject(destination, source)" args := map[string]interface{}{} var srcEncryption, dstEncryption encrypt.ServerSide @@ -7232,10 +7236,9 @@ func testUnencryptedToSSECCopyObject() { // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - var sseSrc encrypt.ServerSide sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject")) // c.TraceOn(os.Stderr) - testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) + testEncryptedCopyObjectWrapper(c, bucketName, nil, sseDst) } // Test encrypted copy object From 8cc0c8c93cd5c1ce230464f5095ca2e42d3d95b9 Mon Sep 17 00:00:00 2001 From: Dustin Guerrero Date: Tue, 21 Jul 2020 12:07:47 -0700 Subject: [PATCH 200/215] Run goimports --- api-error-response.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-error-response.go b/api-error-response.go index 1224a9283..59694614b 100644 --- a/api-error-response.go +++ b/api-error-response.go @@ -113,7 +113,7 @@ func httpRespToErrorResponse(resp *http.Response, bucketName, objectName string) err := xmlDecoder(resp.Body, &errResp) // Xml decoding failed with no body, fall back to HTTP headers. - if err != nil || errResp.Message == "" { + if err != nil || errResp.Message == "" { switch resp.StatusCode { case http.StatusNotFound: if objectName == "" { From e0105ca08252eaef26a21edd4782e2a703f47112 Mon Sep 17 00:00:00 2001 From: Anis Elleuch Date: Wed, 22 Jul 2020 17:23:08 +0100 Subject: [PATCH 201/215] Return object's version id in StatObject() (#1349) This commit returns the content of x-amz-version-id header in the response of the HEAD request, so that information will be part of the returned ObjectInfo structure. A test is added as well. --- constants.go | 2 ++ functional_tests.go | 4 ++++ utils.go | 1 + 3 files changed, 7 insertions(+) diff --git a/constants.go b/constants.go index c87fb3bd0..1c2ea8ef2 100644 --- a/constants.go +++ b/constants.go @@ -66,6 +66,8 @@ const ( amzTaggingHeader = "X-Amz-Tagging" amzTaggingHeaderDirective = "X-Amz-Tagging-Directive" + amzVersionID = "X-Amz-Version-Id" + // Object legal hold header amzLegalHoldHeader = "X-Amz-Object-Lock-Legal-Hold" diff --git a/functional_tests.go b/functional_tests.go index 055735b7f..476072768 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -837,6 +837,10 @@ func testStatObjectWithVersioning() { logError(testName, function, args, startTime, "", "error during HEAD object", err) return } + if statInfo.VersionID == "" || statInfo.VersionID != results[i].VersionID { + logError(testName, function, args, startTime, "", "error during HEAD object, unexpected version id", err) + return + } if statInfo.ETag != results[i].ETag { logError(testName, function, args, startTime, "", "error during HEAD object, unexpected ETag", err) return diff --git a/utils.go b/utils.go index f5400d95a..962635508 100644 --- a/utils.go +++ b/utils.go @@ -272,6 +272,7 @@ func ToObjectInfo(bucketName string, objectName string, h http.Header) (ObjectIn LastModified: date, ContentType: contentType, Expires: expTime, + VersionID: h.Get(amzVersionID), // Extract only the relevant header keys describing the object. // following function filters out a list of standard set of keys // which are not part of object metadata. From 4f0e9a6a0be0f439afa2b5754d1028e309857d61 Mon Sep 17 00:00:00 2001 From: Dustin Guerrero Date: Wed, 22 Jul 2020 11:27:21 -0700 Subject: [PATCH 202/215] Remove github workflows --- .github/workflows/codeql.yml | 52 -------------------------------- .github/workflows/go-windows.yml | 45 --------------------------- .github/workflows/go.yml | 47 ----------------------------- 3 files changed, 144 deletions(-) delete mode 100644 .github/workflows/codeql.yml delete mode 100644 .github/workflows/go-windows.yml delete mode 100644 .github/workflows/go.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index 9771ca0f4..000000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: "Code scanning - action" - -on: - push: - pull_request: - schedule: - - cron: '0 19 * * 0' - -jobs: - CodeQL-Build: - - # CodeQL runs on ubuntu-latest and windows-latest - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - with: - # We must fetch at least the immediate parents so that if this is - # a pull request then we can checkout the head. - fetch-depth: 2 - - # If this run was triggered by a pull request event, then checkout - # the head of the pull request instead of the merge commit. - - run: git checkout HEAD^2 - if: ${{ github.event_name == 'pull_request' }} - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - # Override language selection by uncommenting this and choosing your languages - # with: - # languages: go, javascript, csharp, python, cpp, java - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v1 - - # ℹ️ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/go-windows.yml b/.github/workflows/go-windows.yml deleted file mode 100644 index 91d240691..000000000 --- a/.github/workflows/go-windows.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: Go - -on: - pull_request: - branches: - - master - push: - branches: - - master - -jobs: - build: - name: Test on Go ${{ matrix.go-version }} and ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - go-version: [1.13.x, 1.14.x] - os: [windows-latest] - steps: - - name: Set up Go ${{ matrix.go-version }} on ${{ matrix.os }} - uses: actions/setup-go@v1 - with: - go-version: ${{ matrix.go-version }} - id: go - - - name: Check out code into the Go module directory - uses: actions/checkout@v1 - - - name: Build on ${{ matrix.os }} - env: - MINT_MODE: core - SERVER_ENDPOINT: localhost:9000 - ACCESS_KEY: minio - SECRET_KEY: minio123 - ENABLE_HTTPS: 1 - MINIO_ACCESS_KEY: minio - MINIO_SECRET_KEY: minio123 - GO111MODULE: on - run: | - New-Item -ItemType Directory -Path "$env:temp/certs-dir" - Copy-Item -Path testcerts\* -Destination "$env:temp/certs-dir" - Invoke-WebRequest -Uri https://dl.minio.io/server/minio/release/windows-amd64/minio.exe -OutFile $HOME/minio.exe - Start-Process -NoNewWindow -FilePath "$HOME/minio.exe" -ArgumentList "-S", "$env:temp/certs-dir", "server", "$env:temp/fs{1...4}" - $env:SSL_CERT_FILE = "$env:temp/certs-dir/public.crt" - go run functional_tests.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml deleted file mode 100644 index ccc2a235c..000000000 --- a/.github/workflows/go.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Go - -on: - pull_request: - branches: - - master - push: - branches: - - master - -jobs: - build: - name: Test on Go ${{ matrix.go-version }} and ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - go-version: [1.13.x, 1.14.x] - os: [ubuntu-latest] - steps: - - name: Set up Go ${{ matrix.go-version }} on ${{ matrix.os }} - uses: actions/setup-go@v1 - with: - go-version: ${{ matrix.go-version }} - id: go - - - name: Check out code into the Go module directory - uses: actions/checkout@v1 - - - name: Build on ${{ matrix.os }} - env: - MINT_MODE: full - SERVER_ENDPOINT: localhost:9000 - ACCESS_KEY: minio - SECRET_KEY: minio123 - ENABLE_HTTPS: 1 - MINIO_ACCESS_KEY: minio - MINIO_SECRET_KEY: minio123 - GO111MODULE: on - SSL_CERT_FILE: /tmp/certs-dir/public.crt - run: | - sudo apt-get install devscripts - wget -O /tmp/minio https://dl.min.io/server/minio/release/linux-amd64/minio - chmod +x /tmp/minio - mkdir -p /tmp/certs-dir - cp testcerts/* /tmp/certs-dir - /tmp/minio server --quiet -S /tmp/certs-dir /tmp/fs{1...4} & - make From 4a77d5f120dd0f2e8e8f9c2cc8b6f587114e5d73 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 22 Jul 2020 12:57:22 -0700 Subject: [PATCH 203/215] add replication-status, expiration headers (#1350) extend ObjectInfo, UploadInfo to provide more information presented in the HTTP responses --- .github/workflows/go-windows.yml | 2 +- .github/workflows/go.yml | 2 +- api-datatypes.go | 21 ++++++++++- api-put-object-copy.go | 15 +++++--- api-put-object-multipart.go | 15 +++++--- api-put-object-streaming.go | 15 +++++--- constants.go | 5 ++- utils.go | 61 +++++++++++++++++++++++++------- 8 files changed, 105 insertions(+), 31 deletions(-) diff --git a/.github/workflows/go-windows.yml b/.github/workflows/go-windows.yml index 0e4909c9d..3b04b9408 100644 --- a/.github/workflows/go-windows.yml +++ b/.github/workflows/go-windows.yml @@ -40,7 +40,7 @@ jobs: run: | New-Item -ItemType Directory -Path "$env:temp/certs-dir" Copy-Item -Path testcerts\* -Destination "$env:temp/certs-dir" - Invoke-WebRequest -Uri https://dl.minio.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2020-07-14T19-14-30Z -OutFile $HOME/minio.exe + Invoke-WebRequest -Uri https://dl.minio.io/server/minio/release/windows-amd64/minio.exe -OutFile $HOME/minio.exe Start-Process -NoNewWindow -FilePath "$HOME/minio.exe" -ArgumentList "-S", "$env:temp/certs-dir", "server", "$env:temp/fs{1...4}" $env:SSL_CERT_FILE = "$env:temp/certs-dir/public.crt" go run functional_tests.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index d54fbf021..27d73e657 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -40,7 +40,7 @@ jobs: SSL_CERT_FILE: /tmp/certs-dir/public.crt run: | sudo apt-get install devscripts - wget -O /tmp/minio https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2020-07-14T19-14-30Z + wget -O /tmp/minio https://dl.minio.io/server/minio/release/linux-amd64/minio chmod +x /tmp/minio mkdir -p /tmp/certs-dir cp testcerts/* /tmp/certs-dir diff --git a/api-datatypes.go b/api-datatypes.go index 890b60271..970e1fa5e 100644 --- a/api-datatypes.go +++ b/api-datatypes.go @@ -76,9 +76,13 @@ type UploadInfo struct { ETag string Size int64 LastModified time.Time - Expiration time.Time Location string VersionID string + + // Lifecycle expiry-date and ruleID associated with the expiry + // not to be confused with `Expires` HTTP header. + Expiration time.Time + ExpirationRuleID string } // ObjectInfo container for object metadata. @@ -104,6 +108,9 @@ type ObjectInfo struct { // x-amz-tagging values in their k/v values. UserTags map[string]string `json:"userTags"` + // x-amz-tagging-count value + UserTagCount int + // Owner name. Owner Owner @@ -125,6 +132,18 @@ type ObjectInfo struct { IsDeleteMarker bool VersionID string `xml:"VersionId"` + // x-amz-replication-status value is either in one of the following states + // - COMPLETE + // - PENDING + // - FAILED + // - REPLICA (on the destination) + ReplicationStatus string `xml:"ReplicationStatus"` + + // Lifecycle expiry-date and ruleID associated with the expiry + // not to be confused with `Expires` HTTP header. + Expiration time.Time + ExpirationRuleID string + // Error Err error `json:"-"` } diff --git a/api-put-object-copy.go b/api-put-object-copy.go index 13e17dcf5..9af036ec0 100644 --- a/api-put-object-copy.go +++ b/api-put-object-copy.go @@ -62,11 +62,16 @@ func (c Client) CopyObject(ctx context.Context, dst CopyDestOptions, src CopySrc return UploadInfo{}, err } + // extract lifecycle expiry date and rule ID + expTime, ruleID := amzExpirationToExpiryDateRuleID(resp.Header.Get(amzExpiration)) + return UploadInfo{ - Bucket: dst.Bucket, - Key: dst.Object, - LastModified: cpObjRes.LastModified, - VersionID: resp.Header.Get("x-amz-version-id"), - ETag: trimEtag(resp.Header.Get("ETag")), + Bucket: dst.Bucket, + Key: dst.Object, + LastModified: cpObjRes.LastModified, + ETag: trimEtag(resp.Header.Get("ETag")), + VersionID: resp.Header.Get(amzVersionID), + Expiration: expTime, + ExpirationRuleID: ruleID, }, nil } diff --git a/api-put-object-multipart.go b/api-put-object-multipart.go index 25e633fcf..ed859f930 100644 --- a/api-put-object-multipart.go +++ b/api-put-object-multipart.go @@ -368,12 +368,17 @@ func (c Client) completeMultipartUpload(ctx context.Context, bucketName, objectN return UploadInfo{}, completeMultipartUploadErr } + // extract lifecycle expiry date and rule ID + expTime, ruleID := amzExpirationToExpiryDateRuleID(resp.Header.Get(amzExpiration)) + return UploadInfo{ - Bucket: completeMultipartUploadResult.Bucket, - Key: completeMultipartUploadResult.Key, - ETag: trimEtag(completeMultipartUploadResult.ETag), - VersionID: resp.Header.Get("x-amz-version-id"), - Location: completeMultipartUploadResult.Location, + Bucket: completeMultipartUploadResult.Bucket, + Key: completeMultipartUploadResult.Key, + ETag: trimEtag(completeMultipartUploadResult.ETag), + VersionID: resp.Header.Get(amzVersionID), + Location: completeMultipartUploadResult.Location, + Expiration: expTime, + ExpirationRuleID: ruleID, }, nil } diff --git a/api-put-object-streaming.go b/api-put-object-streaming.go index 144dbb67e..fbc806b6e 100644 --- a/api-put-object-streaming.go +++ b/api-put-object-streaming.go @@ -454,11 +454,16 @@ func (c Client) putObjectDo(ctx context.Context, bucketName, objectName string, } } + // extract lifecycle expiry date and rule ID + expTime, ruleID := amzExpirationToExpiryDateRuleID(resp.Header.Get(amzExpiration)) + return UploadInfo{ - Bucket: bucketName, - Key: objectName, - ETag: trimEtag(resp.Header.Get("ETag")), - Size: size, - VersionID: resp.Header.Get("x-amz-version-id"), + Bucket: bucketName, + Key: objectName, + ETag: trimEtag(resp.Header.Get("ETag")), + VersionID: resp.Header.Get(amzVersionID), + Size: size, + Expiration: expTime, + ExpirationRuleID: ruleID, }, nil } diff --git a/constants.go b/constants.go index 1c2ea8ef2..e1da72e79 100644 --- a/constants.go +++ b/constants.go @@ -66,7 +66,10 @@ const ( amzTaggingHeader = "X-Amz-Tagging" amzTaggingHeaderDirective = "X-Amz-Tagging-Directive" - amzVersionID = "X-Amz-Version-Id" + amzVersionID = "X-Amz-Version-Id" + amzTaggingCount = "X-Amz-Tagging-Count" + amzExpiration = "X-Amz-Expiration" + amzReplicationStatus = "X-Amz-Replication-Status" // Object legal hold header amzLegalHoldHeader = "X-Amz-Object-Lock-Legal-Hold" diff --git a/utils.go b/utils.go index 962635508..b76005b64 100644 --- a/utils.go +++ b/utils.go @@ -22,6 +22,7 @@ import ( "encoding/base64" "encoding/hex" "encoding/xml" + "fmt" "hash" "io" "io/ioutil" @@ -44,6 +45,19 @@ func trimEtag(etag string) string { return strings.TrimSuffix(etag, "\"") } +var expirationRegex = regexp.MustCompile(`expiry-date="(.*?)", rule-id="(.*?)"`) + +func amzExpirationToExpiryDateRuleID(expiration string) (time.Time, string) { + if matches := expirationRegex.FindStringSubmatch(expiration); len(matches) == 3 { + expTime, err := time.Parse(http.TimeFormat, matches[1]) + if err != nil { + return time.Time{}, "" + } + return expTime, matches[2] + } + return time.Time{}, "" +} + // xmlDecoder provide decoded value in xml. func xmlDecoder(body io.Reader, v interface{}) error { d := xml.NewDecoder(body) @@ -219,7 +233,7 @@ func ToObjectInfo(bucketName string, objectName string, h http.Header) (ObjectIn // Content-Length is not valid return ObjectInfo{}, ErrorResponse{ Code: "InternalError", - Message: "Content-Length is invalid. " + reportIssue, + Message: fmt.Sprintf("Content-Length is not an integer, failed with %v", err), BucketName: bucketName, Key: objectName, RequestID: h.Get("x-amz-request-id"), @@ -234,7 +248,7 @@ func ToObjectInfo(bucketName string, objectName string, h http.Header) (ObjectIn if err != nil { return ObjectInfo{}, ErrorResponse{ Code: "InternalError", - Message: "Last-Modified time format is invalid. " + reportIssue, + Message: fmt.Sprintf("Last-Modified time format is invalid, failed with %v", err), BucketName: bucketName, Key: objectName, RequestID: h.Get("x-amz-request-id"), @@ -250,9 +264,9 @@ func ToObjectInfo(bucketName string, objectName string, h http.Header) (ObjectIn } expiryStr := h.Get("Expires") - var expTime time.Time - if t, err := time.Parse(http.TimeFormat, expiryStr); err == nil { - expTime = t.UTC() + var expiry time.Time + if expiryStr != "" { + expiry, _ = time.Parse(http.TimeFormat, expiryStr) } metadata := extractObjMetadata(h) @@ -264,21 +278,44 @@ func ToObjectInfo(bucketName string, objectName string, h http.Header) (ObjectIn } userTags := s3utils.TagDecode(h.Get(amzTaggingHeader)) + var tagCount int + if count := h.Get(amzTaggingCount); count != "" { + tagCount, err = strconv.Atoi(count) + if err != nil { + return ObjectInfo{}, ErrorResponse{ + Code: "InternalError", + Message: fmt.Sprintf("x-amz-tagging-count is not an integer, failed with %v", err), + BucketName: bucketName, + Key: objectName, + RequestID: h.Get("x-amz-request-id"), + HostID: h.Get("x-amz-id-2"), + Region: h.Get("x-amz-bucket-region"), + } + } + } + + // extract lifecycle expiry date and rule ID + expTime, ruleID := amzExpirationToExpiryDateRuleID(h.Get(amzExpiration)) + // Save object metadata info. return ObjectInfo{ - ETag: etag, - Key: objectName, - Size: size, - LastModified: date, - ContentType: contentType, - Expires: expTime, - VersionID: h.Get(amzVersionID), + ETag: etag, + Key: objectName, + Size: size, + LastModified: date, + ContentType: contentType, + Expires: expiry, + VersionID: h.Get(amzVersionID), + ReplicationStatus: h.Get(amzReplicationStatus), + Expiration: expTime, + ExpirationRuleID: ruleID, // Extract only the relevant header keys describing the object. // following function filters out a list of standard set of keys // which are not part of object metadata. Metadata: metadata, UserMetadata: userMetadata, UserTags: userTags, + UserTagCount: tagCount, }, nil } From d4c96d5b97533826c3dc5874944a45a0e068b65e Mon Sep 17 00:00:00 2001 From: Anis Elleuch Date: Thu, 23 Jul 2020 19:23:10 +0100 Subject: [PATCH 204/215] lifecycle: Fix marshaling expiration date/days (#1351) --- docs/API.md | 10 ++++++---- examples/s3/setbucketlifecycle.go | 10 ++++++---- pkg/lifecycle/lifecycle.go | 10 +++++----- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/docs/API.md b/docs/API.md index 2ac72c8f2..5fb16a804 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1578,10 +1578,12 @@ __Example__ ```go config := lifecycle.NewConfiguration() config.Rules = []lifecycle.Rule{ - ID: "expire-bucket", - Status: "Enabled", - Expiration: lifecycle.Expiration{ - Days: 365, + { + ID: "expire-bucket", + Status: "Enabled", + Expiration: lifecycle.Expiration{ + Days: 365, + }, }, } diff --git a/examples/s3/setbucketlifecycle.go b/examples/s3/setbucketlifecycle.go index 2f4869d8d..f829f8ea4 100644 --- a/examples/s3/setbucketlifecycle.go +++ b/examples/s3/setbucketlifecycle.go @@ -50,10 +50,12 @@ func main() { // Set lifecycle on a bucket config := lifecycle.NewConfiguration() config.Rules = []lifecycle.Rule{ - ID: "expire-bucket", - Status: "Enabled", - Expiration: lifecycle.Expiration{ - Days: 365, + { + ID: "expire-bucket", + Status: "Enabled", + Expiration: lifecycle.Expiration{ + Days: 365, + }, }, } err = s3Client.SetBucketLifecycle(context.Background(), "my-bucketname", config) diff --git a/pkg/lifecycle/lifecycle.go b/pkg/lifecycle/lifecycle.go index 4dfa110e6..3493003f7 100644 --- a/pkg/lifecycle/lifecycle.go +++ b/pkg/lifecycle/lifecycle.go @@ -180,11 +180,11 @@ type ExpirationDays int // MarshalXML encodes number of days to expire if it is non-zero and // encodes empty string otherwise -func (eDays *ExpirationDays) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { - if *eDays == ExpirationDays(0) { +func (eDays ExpirationDays) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { + if eDays == 0 { return nil } - return e.EncodeElement(int(*eDays), startElement) + return e.EncodeElement(int(eDays), startElement) } // ExpirationDate is a embedded type containing time.Time to unmarshal @@ -195,8 +195,8 @@ type ExpirationDate struct { // MarshalXML encodes expiration date if it is non-zero and encodes // empty string otherwise -func (eDate *ExpirationDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { - if *eDate == (ExpirationDate{time.Time{}}) { +func (eDate ExpirationDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { + if eDate.Time.IsZero() { return nil } return e.EncodeElement(eDate.Format(time.RFC3339), startElement) From 213399374df22083063010b8effe92ba863458ca Mon Sep 17 00:00:00 2001 From: Julien K <60636190+juli3nk@users.noreply.github.com> Date: Fri, 24 Jul 2020 11:34:11 -0400 Subject: [PATCH 205/215] fix a typo in README.md (#1352) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7c2b6d78c..182d961ad 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ func main() { bucketName := "mymusic" location := "us-east-1" - err = minioClient.MakeBucket(context.Background(), bucketName, MakeBucketOptions{Region: location}) + err = minioClient.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: location}) if err != nil { // Check to see if we already own this bucket (which happens if you run this twice) exists, errBucketExists := minioClient.BucketExists(bucketName) From f14ef11d4858fafc34cd11f69083057817124b63 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Sun, 26 Jul 2020 07:51:40 +0000 Subject: [PATCH 206/215] Update version to next release --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index f3d9929fc..55de7c2a3 100644 --- a/api.go +++ b/api.go @@ -108,7 +108,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v7.0.2" + libraryVersion = "v7.0.3" ) // User Agent should always following the below style. From 0971bb12997770381128d2af22fb4e3ca71de20e Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 27 Jul 2020 14:44:30 -0700 Subject: [PATCH 207/215] fix: handle readFull bug with certain readers (#1354) --- api-get-object.go | 6 +-- api-put-object-multipart.go | 7 ++-- api-put-object-streaming.go | 10 +++-- api-put-object.go | 4 +- api-select.go | 6 +-- core_test.go | 15 ++++--- functional_tests.go | 79 ++++++++++++++++++++++++++----------- utils.go | 35 ++++++++++++++++ 8 files changed, 118 insertions(+), 44 deletions(-) diff --git a/api-get-object.go b/api-get-object.go index ac9bbdbe0..2df1112a9 100644 --- a/api-get-object.go +++ b/api-get-object.go @@ -102,7 +102,7 @@ func (c Client) GetObject(ctx context.Context, bucketName, objectName string, op etag = objectInfo.ETag // Read at least firstReq.Buffer bytes, if not we have // reached our EOF. - size, err := io.ReadFull(httpReader, req.Buffer) + size, err := readFull(httpReader, req.Buffer) if size > 0 && err == io.ErrUnexpectedEOF { // If an EOF happens after reading some but not // all the bytes ReadFull returns ErrUnexpectedEOF @@ -192,8 +192,8 @@ func (c Client) GetObject(ctx context.Context, bucketName, objectName string, op // Read at least req.Buffer bytes, if not we have // reached our EOF. - size, err := io.ReadFull(httpReader, req.Buffer) - if err == io.ErrUnexpectedEOF { + size, err := readFull(httpReader, req.Buffer) + if size > 0 && err == io.ErrUnexpectedEOF { // If an EOF happens after reading some but not // all the bytes ReadFull returns ErrUnexpectedEOF err = io.EOF diff --git a/api-put-object-multipart.go b/api-put-object-multipart.go index ed859f930..1c862ad96 100644 --- a/api-put-object-multipart.go +++ b/api-put-object-multipart.go @@ -104,11 +104,12 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje // HTTPS connection. hashAlgos, hashSums := c.hashMaterials(opts.SendContentMd5) - length, rErr := io.ReadFull(reader, buf) - if rErr == io.EOF { + length, rErr := readFull(reader, buf) + if rErr == io.EOF && partNumber > 1 { break } - if rErr != nil && rErr != io.ErrUnexpectedEOF { + + if rErr != nil && rErr != io.ErrUnexpectedEOF && rErr != io.EOF { return UploadInfo{}, rErr } diff --git a/api-put-object-streaming.go b/api-put-object-streaming.go index fbc806b6e..3ef2e8b22 100644 --- a/api-put-object-streaming.go +++ b/api-put-object-streaming.go @@ -279,13 +279,15 @@ func (c Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, bu } if opts.SendContentMd5 { - length, rerr := io.ReadFull(reader, buf) + length, rerr := readFull(reader, buf) if rerr == io.EOF && partNumber > 1 { break } - if rerr != nil && rerr != io.ErrUnexpectedEOF && rerr != io.EOF { + + if rerr != nil && rerr != io.ErrUnexpectedEOF && err != io.EOF { return UploadInfo{}, rerr } + // Calculate md5sum. hash := c.md5Hasher() hash.Write(buf[:length]) @@ -389,8 +391,8 @@ func (c Client) putObject(ctx context.Context, bucketName, objectName string, re // Create a buffer. buf := make([]byte, size) - length, rErr := io.ReadFull(reader, buf) - if rErr != nil && rErr != io.ErrUnexpectedEOF { + length, rErr := readFull(reader, buf) + if rErr != nil && rErr != io.ErrUnexpectedEOF && rErr != io.EOF { return UploadInfo{}, rErr } diff --git a/api-put-object.go b/api-put-object.go index 8cbe004d2..b0d5af466 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -230,6 +230,7 @@ func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName stri } return c.putObjectMultipart(ctx, bucketName, objectName, reader, size, opts) } + if size < 0 { return c.putObjectMultipartStreamNoLength(ctx, bucketName, objectName, reader, opts) } @@ -284,10 +285,11 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName buf := make([]byte, partSize) for partNumber <= totalPartsCount { - length, rerr := io.ReadFull(reader, buf) + length, rerr := readFull(reader, buf) if rerr == io.EOF && partNumber > 1 { break } + if rerr != nil && rerr != io.ErrUnexpectedEOF && rerr != io.EOF { return UploadInfo{}, rerr } diff --git a/api-select.go b/api-select.go index 340fa5299..e35cf02bf 100644 --- a/api-select.go +++ b/api-select.go @@ -709,7 +709,7 @@ func extractString(source io.Reader, lenBytes int) (string, error) { // extractUint32 extracts a 4 byte integer from the byte array. func extractUint32(r io.Reader) (uint32, error) { buf := make([]byte, 4) - _, err := io.ReadFull(r, buf) + _, err := readFull(r, buf) if err != nil { return 0, err } @@ -719,7 +719,7 @@ func extractUint32(r io.Reader) (uint32, error) { // extractUint16 extracts a 2 byte integer from the byte array. func extractUint16(r io.Reader) (uint16, error) { buf := make([]byte, 2) - _, err := io.ReadFull(r, buf) + _, err := readFull(r, buf) if err != nil { return 0, err } @@ -729,7 +729,7 @@ func extractUint16(r io.Reader) (uint16, error) { // extractUint8 extracts a 1 byte integer from the byte array. func extractUint8(r io.Reader) (uint8, error) { buf := make([]byte, 1) - _, err := io.ReadFull(r, buf) + _, err := readFull(r, buf) if err != nil { return 0, err } diff --git a/core_test.go b/core_test.go index 0279f6bb3..7da03ed2d 100644 --- a/core_test.go +++ b/core_test.go @@ -20,7 +20,6 @@ package minio import ( "bytes" "context" - "io" "log" "net/http" "os" @@ -134,7 +133,7 @@ func TestGetObjectCore(t *testing.T) { if err != nil { t.Fatal(err) } - m, err := io.ReadFull(reader, buf1) + m, err := readFull(reader, buf1) reader.Close() if err != nil { t.Fatal(err) @@ -154,7 +153,7 @@ func TestGetObjectCore(t *testing.T) { t.Fatal(err) } - m, err = io.ReadFull(reader, buf2) + m, err = readFull(reader, buf2) reader.Close() if err != nil { t.Fatal(err) @@ -173,7 +172,7 @@ func TestGetObjectCore(t *testing.T) { t.Fatal(err) } - m, err = io.ReadFull(reader, buf3) + m, err = readFull(reader, buf3) if err != nil { reader.Close() t.Fatal(err) @@ -204,7 +203,7 @@ func TestGetObjectCore(t *testing.T) { t.Fatal(err) } - m, err = io.ReadFull(reader, buf3) + m, err = readFull(reader, buf3) reader.Close() if err != nil { t.Fatal(err) @@ -224,7 +223,7 @@ func TestGetObjectCore(t *testing.T) { t.Fatal(err) } - m, err = io.ReadFull(reader, buf4) + m, err = readFull(reader, buf4) reader.Close() if err != nil { t.Fatal(err) @@ -634,7 +633,7 @@ func TestCoreCopyObjectPart(t *testing.T) { t.Fatal("Error:", err, destBucketName, destObjectName) } getBuf := make([]byte, 5*1024*1024) - _, err = io.ReadFull(r, getBuf) + _, err = readFull(r, getBuf) if err != nil { t.Fatal("Error:", err, destBucketName, destObjectName) } @@ -648,7 +647,7 @@ func TestCoreCopyObjectPart(t *testing.T) { t.Fatal("Error:", err, destBucketName, destObjectName) } getBuf = make([]byte, 5*1024*1024+1) - _, err = io.ReadFull(r, getBuf) + _, err = readFull(r, getBuf) if err != nil { t.Fatal("Error:", err, destBucketName, destObjectName) } diff --git a/functional_tests.go b/functional_tests.go index 476072768..9099efe3b 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -86,6 +86,41 @@ func (f *mintJSONFormatter) Format(entry *log.Entry) ([]byte, error) { return append(serialized, '\n'), nil } +var readFull = func(r io.Reader, buf []byte) (n int, err error) { + // ReadFull reads exactly len(buf) bytes from r into buf. + // It returns the number of bytes copied and an error if + // fewer bytes were read. The error is EOF only if no bytes + // were read. If an EOF happens after reading some but not + // all the bytes, ReadFull returns ErrUnexpectedEOF. + // On return, n == len(buf) if and only if err == nil. + // If r returns an error having read at least len(buf) bytes, + // the error is dropped. + for n < len(buf) && err == nil { + var nn int + nn, err = r.Read(buf[n:]) + // Some spurious io.Reader's return + // io.ErrUnexpectedEOF when nn == 0 + // this behavior is undocumented + // so we are on purpose not using io.ReadFull + // implementation because this can lead + // to custom handling, to avoid that + // we simply modify the original io.ReadFull + // implementation to avoid this issue. + // io.ErrUnexpectedEOF with nn == 0 really + // means that io.EOF + if err == io.ErrUnexpectedEOF && nn == 0 { + err = io.EOF + } + n += nn + } + if n >= len(buf) { + err = nil + } else if n > 0 && err == io.EOF { + err = io.ErrUnexpectedEOF + } + return +} + func cleanEmptyEntries(fields log.Fields) log.Fields { cleanFields := log.Fields{} for k, v := range fields { @@ -1967,9 +2002,9 @@ func testGetObjectSeekEnd() { return } buf2 := make([]byte, 100) - m, err := io.ReadFull(r, buf2) + m, err := readFull(r, buf2) if err != nil { - logError(testName, function, args, startTime, "", "Error reading through io.ReadFull", err) + logError(testName, function, args, startTime, "", "Error reading through readFull", err) return } if m != len(buf2) { @@ -7684,7 +7719,7 @@ func testSSECMultipartEncryptedToSSECCopyObjectPart() { logError(testName, function, args, startTime, "", "GetObject call failed", err) } getBuf := make([]byte, 6*1024*1024) - _, err = io.ReadFull(r, getBuf) + _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) } @@ -7698,7 +7733,7 @@ func testSSECMultipartEncryptedToSSECCopyObjectPart() { logError(testName, function, args, startTime, "", "GetObject call failed", err) } getBuf = make([]byte, 6*1024*1024+1) - _, err = io.ReadFull(r, getBuf) + _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) } @@ -7843,7 +7878,7 @@ func testSSECEncryptedToSSECCopyObjectPart() { logError(testName, function, args, startTime, "", "GetObject call failed", err) } getBuf := make([]byte, 5*1024*1024) - _, err = io.ReadFull(r, getBuf) + _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) } @@ -7857,7 +7892,7 @@ func testSSECEncryptedToSSECCopyObjectPart() { logError(testName, function, args, startTime, "", "GetObject call failed", err) } getBuf = make([]byte, 5*1024*1024+1) - _, err = io.ReadFull(r, getBuf) + _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) } @@ -8001,7 +8036,7 @@ func testSSECEncryptedToUnencryptedCopyPart() { logError(testName, function, args, startTime, "", "GetObject call failed", err) } getBuf := make([]byte, 5*1024*1024) - _, err = io.ReadFull(r, getBuf) + _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) } @@ -8015,7 +8050,7 @@ func testSSECEncryptedToUnencryptedCopyPart() { logError(testName, function, args, startTime, "", "GetObject call failed", err) } getBuf = make([]byte, 5*1024*1024+1) - _, err = io.ReadFull(r, getBuf) + _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) } @@ -8162,7 +8197,7 @@ func testSSECEncryptedToSSES3CopyObjectPart() { logError(testName, function, args, startTime, "", "GetObject call failed", err) } getBuf := make([]byte, 5*1024*1024) - _, err = io.ReadFull(r, getBuf) + _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) } @@ -8176,7 +8211,7 @@ func testSSECEncryptedToSSES3CopyObjectPart() { logError(testName, function, args, startTime, "", "GetObject call failed", err) } getBuf = make([]byte, 5*1024*1024+1) - _, err = io.ReadFull(r, getBuf) + _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) } @@ -8318,7 +8353,7 @@ func testUnencryptedToSSECCopyObjectPart() { logError(testName, function, args, startTime, "", "GetObject call failed", err) } getBuf := make([]byte, 5*1024*1024) - _, err = io.ReadFull(r, getBuf) + _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) } @@ -8332,7 +8367,7 @@ func testUnencryptedToSSECCopyObjectPart() { logError(testName, function, args, startTime, "", "GetObject call failed", err) } getBuf = make([]byte, 5*1024*1024+1) - _, err = io.ReadFull(r, getBuf) + _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) } @@ -8470,7 +8505,7 @@ func testUnencryptedToUnencryptedCopyPart() { logError(testName, function, args, startTime, "", "GetObject call failed", err) } getBuf := make([]byte, 5*1024*1024) - _, err = io.ReadFull(r, getBuf) + _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) } @@ -8484,7 +8519,7 @@ func testUnencryptedToUnencryptedCopyPart() { logError(testName, function, args, startTime, "", "GetObject call failed", err) } getBuf = make([]byte, 5*1024*1024+1) - _, err = io.ReadFull(r, getBuf) + _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) } @@ -8624,7 +8659,7 @@ func testUnencryptedToSSES3CopyObjectPart() { logError(testName, function, args, startTime, "", "GetObject call failed", err) } getBuf := make([]byte, 5*1024*1024) - _, err = io.ReadFull(r, getBuf) + _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) } @@ -8638,7 +8673,7 @@ func testUnencryptedToSSES3CopyObjectPart() { logError(testName, function, args, startTime, "", "GetObject call failed", err) } getBuf = make([]byte, 5*1024*1024+1) - _, err = io.ReadFull(r, getBuf) + _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) } @@ -8781,7 +8816,7 @@ func testSSES3EncryptedToSSECCopyObjectPart() { logError(testName, function, args, startTime, "", "GetObject call failed", err) } getBuf := make([]byte, 5*1024*1024) - _, err = io.ReadFull(r, getBuf) + _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) } @@ -8795,7 +8830,7 @@ func testSSES3EncryptedToSSECCopyObjectPart() { logError(testName, function, args, startTime, "", "GetObject call failed", err) } getBuf = make([]byte, 5*1024*1024+1) - _, err = io.ReadFull(r, getBuf) + _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) } @@ -8934,7 +8969,7 @@ func testSSES3EncryptedToUnencryptedCopyPart() { logError(testName, function, args, startTime, "", "GetObject call failed", err) } getBuf := make([]byte, 5*1024*1024) - _, err = io.ReadFull(r, getBuf) + _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) } @@ -8948,7 +8983,7 @@ func testSSES3EncryptedToUnencryptedCopyPart() { logError(testName, function, args, startTime, "", "GetObject call failed", err) } getBuf = make([]byte, 5*1024*1024+1) - _, err = io.ReadFull(r, getBuf) + _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) } @@ -9090,7 +9125,7 @@ func testSSES3EncryptedToSSES3CopyObjectPart() { logError(testName, function, args, startTime, "", "GetObject call failed", err) } getBuf := make([]byte, 5*1024*1024) - _, err = io.ReadFull(r, getBuf) + _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) } @@ -9104,7 +9139,7 @@ func testSSES3EncryptedToSSES3CopyObjectPart() { logError(testName, function, args, startTime, "", "GetObject call failed", err) } getBuf = make([]byte, 5*1024*1024+1) - _, err = io.ReadFull(r, getBuf) + _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) } diff --git a/utils.go b/utils.go index b76005b64..f1b711d22 100644 --- a/utils.go +++ b/utils.go @@ -319,6 +319,41 @@ func ToObjectInfo(bucketName string, objectName string, h http.Header) (ObjectIn }, nil } +var readFull = func(r io.Reader, buf []byte) (n int, err error) { + // ReadFull reads exactly len(buf) bytes from r into buf. + // It returns the number of bytes copied and an error if + // fewer bytes were read. The error is EOF only if no bytes + // were read. If an EOF happens after reading some but not + // all the bytes, ReadFull returns ErrUnexpectedEOF. + // On return, n == len(buf) if and only if err == nil. + // If r returns an error having read at least len(buf) bytes, + // the error is dropped. + for n < len(buf) && err == nil { + var nn int + nn, err = r.Read(buf[n:]) + // Some spurious io.Reader's return + // io.ErrUnexpectedEOF when nn == 0 + // this behavior is undocumented + // so we are on purpose not using io.ReadFull + // implementation because this can lead + // to custom handling, to avoid that + // we simply modify the original io.ReadFull + // implementation to avoid this issue. + // io.ErrUnexpectedEOF with nn == 0 really + // means that io.EOF + if err == io.ErrUnexpectedEOF && nn == 0 { + err = io.EOF + } + n += nn + } + if n >= len(buf) { + err = nil + } else if n > 0 && err == io.EOF { + err = io.ErrUnexpectedEOF + } + return +} + // regCred matches credential string in HTTP header var regCred = regexp.MustCompile("Credential=([A-Z0-9]+)/") From bbe24e462270eab37c8a3892c832f30692213d04 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sun, 2 Aug 2020 16:57:50 -0700 Subject: [PATCH 208/215] Create CNAME --- CNAME | 1 + 1 file changed, 1 insertion(+) create mode 100644 CNAME diff --git a/CNAME b/CNAME new file mode 100644 index 000000000..d365a7bb2 --- /dev/null +++ b/CNAME @@ -0,0 +1 @@ +minio-go.min.io \ No newline at end of file From a9a9064ee455453707a9431cb0de175c3d85f945 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sun, 2 Aug 2020 16:59:35 -0700 Subject: [PATCH 209/215] remove deprecated build badges --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 182d961ad..0c83e9efb 100644 --- a/README.md +++ b/README.md @@ -248,8 +248,5 @@ The full API Reference is available here. ## Contribute [Contributors Guide](https://github.com/minio/minio-go/blob/master/CONTRIBUTING.md) -[![Build Status](https://travis-ci.org/minio/minio-go.svg)](https://travis-ci.org/minio/minio-go) -[![Build status](https://ci.appveyor.com/api/projects/status/1d05e6nvxcelmrak?svg=true)](https://ci.appveyor.com/project/harshavardhana/minio-go) - ## License This SDK is distributed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0), see [LICENSE](./LICENSE) and [NOTICE](./NOTICE) for more information. From f0c6c8f5d5989b05f3e2d82fdc2d574e43cd8f7c Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 4 Aug 2020 09:30:04 -0700 Subject: [PATCH 210/215] fix: avoid data race by copying the buffer (#1357) fixes #1355 --- api-put-object-streaming.go | 25 ++++++++++++++++++++----- api.go | 10 +++++----- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/api-put-object-streaming.go b/api-put-object-streaming.go index 3ef2e8b22..251fee9f8 100644 --- a/api-put-object-streaming.go +++ b/api-put-object-streaming.go @@ -146,9 +146,15 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa uploadPartsCh <- uploadPartReq{PartNum: p} } close(uploadPartsCh) + + var partsBuf = make([][]byte, opts.getNumThreads()) + for i := range partsBuf { + partsBuf[i] = make([]byte, partSize) + } + // Receive each part number from the channel allowing three parallel uploads. for w := 1; w <= opts.getNumThreads(); w++ { - go func(partSize int64) { + go func(w int, partSize int64) { // Each worker will draw from the part channel and upload in parallel. for uploadReq := range uploadPartsCh { @@ -164,12 +170,21 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa partSize = lastPartSize } + n, rerr := readFull(io.NewSectionReader(reader, readOffset, partSize), partsBuf[w-1][:partSize]) + if rerr != nil && rerr != io.ErrUnexpectedEOF && err != io.EOF { + uploadedPartsCh <- uploadedPartRes{ + Error: rerr, + } + // Exit the goroutine. + return + } + // Get a section reader on a particular offset. - sectionReader := newHook(io.NewSectionReader(reader, readOffset, partSize), opts.Progress) + hookReader := newHook(bytes.NewReader(partsBuf[w-1][:n]), opts.Progress) // Proceed to upload the part. - objPart, err := c.uploadPart(ctx, bucketName, objectName, uploadID, - sectionReader, uploadReq.PartNum, + objPart, err := c.uploadPart(ctx, bucketName, objectName, + uploadID, hookReader, uploadReq.PartNum, "", "", partSize, opts.ServerSideEncryption) if err != nil { uploadedPartsCh <- uploadedPartRes{ @@ -189,7 +204,7 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa Part: uploadReq.Part, } } - }(partSize) + }(w, partSize) } // Gather the responses as they occur and update any diff --git a/api.go b/api.go index 55de7c2a3..09b1e0bb1 100644 --- a/api.go +++ b/api.go @@ -513,7 +513,7 @@ var successStatus = []int{ // request upon any error up to maxRetries attempts in a binomially // delayed manner using a standard back off algorithm. func (c Client) executeMethod(ctx context.Context, method string, metadata requestMetadata) (res *http.Response, err error) { - var isRetryable bool // Indicates if request can be retried. + var retryable bool // Indicates if request can be retried. var bodySeeker io.Seeker // Extracted seeker from io.Reader. var reqRetry = MaxRetry // Indicates how many times we can retry the request @@ -526,13 +526,13 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque if metadata.contentBody != nil { // Check if body is seekable then it is retryable. - bodySeeker, isRetryable = metadata.contentBody.(io.Seeker) + bodySeeker, retryable = metadata.contentBody.(io.Seeker) switch bodySeeker { case os.Stdin, os.Stdout, os.Stderr: - isRetryable = false + retryable = false } // Retry only when reader is seekable - if !isRetryable { + if !retryable { reqRetry = 1 } @@ -559,7 +559,7 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque // error until maxRetries have been exhausted, retry attempts are // performed after waiting for a given period of time in a // binomial fashion. - if isRetryable { + if retryable { // Seek back to beginning for each attempt. if _, err = bodySeeker.Seek(0, 0); err != nil { // If seek failed, no need to retry. From 0a14c1dd2adfce55c2be4c5e3bd0d3936f50dcad Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 4 Aug 2020 11:09:36 -0700 Subject: [PATCH 211/215] fix: only SSE-C headers should be applied to destination (#1359) closes #1358 --- api-compose-object.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-compose-object.go b/api-compose-object.go index cb2e899e5..e10737558 100644 --- a/api-compose-object.go +++ b/api-compose-object.go @@ -454,7 +454,7 @@ func (c Client) ComposeObject(ctx context.Context, dst CopyDestOptions, srcs ... for i, src := range srcs { var h = make(http.Header) src.Marshal(h) - if dst.Encryption != nil { + if dst.Encryption != nil && dst.Encryption.Type() == encrypt.SSEC { dst.Encryption.Marshal(h) } From 0670669b153716bea6cec3271568dab8960deb23 Mon Sep 17 00:00:00 2001 From: poornas Date: Tue, 4 Aug 2020 12:11:54 -0700 Subject: [PATCH 212/215] Add methods to set/remove replication rules (#1356) --- examples/s3/setbucketreplication.go | 2 +- go.mod | 17 +- go.sum | 45 +++-- pkg/replication/replication.go | 271 +++++++++++++++++++++++++++- 4 files changed, 315 insertions(+), 20 deletions(-) diff --git a/examples/s3/setbucketreplication.go b/examples/s3/setbucketreplication.go index 8e21079e8..62437cded 100644 --- a/examples/s3/setbucketreplication.go +++ b/examples/s3/setbucketreplication.go @@ -56,7 +56,7 @@ func main() { } // This replication ARN should have been generated for replication endpoint using `mc admin bucket remote` command - replCfg.ReplicationARN = "arn:minio:s3::dadddae7-f1d7-440f-b5d6-651aa9a8c8a7:*" + replCfg.ReplicationARN = "arn:minio:replica::dadddae7-f1d7-440f-b5d6-651aa9a8c8a7:dest" // Set replication config on a bucket err = s3Client.SetBucketReplication(context.Background(), "my-bucketname", replCfg) if err != nil { diff --git a/go.mod b/go.mod index 6d0c73708..448b4bbf1 100644 --- a/go.mod +++ b/go.mod @@ -3,16 +3,23 @@ module github.com/minio/minio-go/v7 go 1.12 require ( - github.com/dustin/go-humanize v1.0.0 // indirect github.com/google/uuid v1.1.1 github.com/json-iterator/go v1.1.10 + github.com/klauspost/cpuid v1.3.1 // indirect + github.com/kr/pretty v0.1.0 // indirect github.com/minio/md5-simd v1.1.0 github.com/minio/sha256-simd v0.1.1 github.com/mitchellh/go-homedir v1.1.0 - github.com/sirupsen/logrus v1.6.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/rs/xid v1.2.1 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect - golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f - golang.org/x/net v0.0.0-20190522155817-f3200d17e092 - golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae // indirect + github.com/stretchr/testify v1.4.0 // indirect + golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 + golang.org/x/net v0.0.0-20200707034311-ab3426394381 + golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect + golang.org/x/text v0.3.3 // indirect + gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/ini.v1 v1.57.0 + gopkg.in/yaml.v2 v2.2.8 // indirect ) diff --git a/go.sum b/go.sum index df4b74773..1858b6ff3 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -14,7 +12,13 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= +github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= @@ -23,35 +27,50 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo= -golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/replication/replication.go b/pkg/replication/replication.go index f21720269..11ec67982 100644 --- a/pkg/replication/replication.go +++ b/pkg/replication/replication.go @@ -16,7 +16,58 @@ package replication -import "encoding/xml" +import ( + "bytes" + "encoding/xml" + "fmt" + "strings" + "unicode/utf8" + + "github.com/rs/xid" +) + +var errInvalidFilter = fmt.Errorf("Invalid filter") + +// OptionType specifies operation to be performed on config +type OptionType string + +const ( + // AddOption specifies addition of rule to config + AddOption OptionType = "Add" + // RemoveOption specifies rule options are for removing a rule + RemoveOption OptionType = "Remove" + // ImportOption is for getting current config + ImportOption OptionType = "Import" +) + +// Options represents options to set a replication configuration rule +type Options struct { + Op OptionType + ID string + Prefix string + Disable bool + Priority int + TagString string + StorageClass string + Arn string +} + +// Tags returns a slice of tags for a rule +func (opts Options) Tags() []Tag { + var tagList []Tag + tagTokens := strings.Split(opts.TagString, "&") + for _, tok := range tagTokens { + if tok == "" { + break + } + kv := strings.SplitN(tok, "=", 2) + tagList = append(tagList, Tag{ + Key: kv[0], + Value: kv[1], + }) + } + return tagList +} // Config - replication configuration specified in // https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html @@ -32,6 +83,97 @@ func (c *Config) Empty() bool { return len(c.Rules) == 0 } +// AddRule adds a new rule to existing replication config. If a rule exists with the +// same ID, then the rule is replaced. +func (c *Config) AddRule(opts Options) error { + tags := opts.Tags() + andVal := And{ + Tags: opts.Tags(), + } + filter := Filter{Prefix: opts.Prefix} + // only a single tag is set. + if opts.Prefix == "" && len(tags) == 1 { + filter.Tag = tags[0] + } + // both prefix and tag are present + if len(andVal.Tags) > 1 || opts.Prefix != "" { + filter.And = andVal + filter.And.Prefix = opts.Prefix + filter.Prefix = "" + } + if opts.ID == "" { + opts.ID = xid.New().String() + } + status := Enabled + if opts.Disable { + status = Disabled + } + tokens := strings.Split(opts.Arn, ":") + if len(tokens) != 6 { + return fmt.Errorf("invalid format for replication Arn") + } + if c.ReplicationARN == "" { // for new configurations + c.ReplicationARN = opts.Arn + } + newRule := Rule{ + ID: opts.ID, + Priority: opts.Priority, + Status: status, + Filter: filter, + Destination: Destination{ + Bucket: fmt.Sprintf("arn:aws:s3:::%s", tokens[5]), + StorageClass: opts.StorageClass, + }, + DeleteMarkerReplication: DeleteMarkerReplication{Status: Disabled}, + } + + ruleFound := false + for i, rule := range c.Rules { + if rule.Priority == newRule.Priority && rule.ID != newRule.ID { + return fmt.Errorf("Priority must be unique. Replication configuration already has a rule with this priority") + } + if rule.Destination.Bucket != newRule.Destination.Bucket { + return fmt.Errorf("The destination bucket must be same for all rules") + } + if rule.ID != newRule.ID { + continue + } + if newRule.Status == Disabled && rule.ID == newRule.ID { + // inherit priority from existing rule, required field on server + newRule.Priority = rule.Priority + } + c.Rules[i] = newRule + ruleFound = true + break + } + // validate rule after overlaying priority for pre-existing rule being disabled. + if err := newRule.Validate(); err != nil { + return err + } + + if !ruleFound { + c.Rules = append(c.Rules, newRule) + } + return nil +} + +// RemoveRule removes a rule from replication config. +func (c *Config) RemoveRule(opts Options) error { + var newRules []Rule + for _, rule := range c.Rules { + if rule.ID != opts.ID { + newRules = append(newRules, rule) + } + } + + if len(newRules) == 0 { + return fmt.Errorf("Replication configuration should have at least one rule") + } + c.Rules = newRules + return nil + +} + // Rule - a rule for replication configuration. type Rule struct { XMLName xml.Name `xml:"Rule" json:"-"` @@ -43,11 +185,112 @@ type Rule struct { Filter Filter `xml:"Filter" json:"Filter"` } +// Validate validates the rule for correctness +func (r Rule) Validate() error { + if err := r.validateID(); err != nil { + return err + } + if err := r.validateStatus(); err != nil { + return err + } + if err := r.validateFilter(); err != nil { + return err + } + + if r.Priority <= 0 && r.Status == Enabled { + return fmt.Errorf("Priority must be set for the rule") + } + + return nil +} + +// validateID - checks if ID is valid or not. +func (r Rule) validateID() error { + // cannot be longer than 255 characters + if len(r.ID) > 255 { + return fmt.Errorf("ID must be less than 255 characters") + } + return nil +} + +// validateStatus - checks if status is valid or not. +func (r Rule) validateStatus() error { + // Status can't be empty + if len(r.Status) == 0 { + return fmt.Errorf("status cannot be empty") + } + + // Status must be one of Enabled or Disabled + if r.Status != Enabled && r.Status != Disabled { + return fmt.Errorf("status must be set to either Enabled or Disabled") + } + return nil +} + +func (r Rule) validateFilter() error { + if err := r.Filter.Validate(); err != nil { + return err + } + return nil +} + +// Prefix - a rule can either have prefix under or under +// . This method returns the prefix from the +// location where it is available +func (r Rule) Prefix() string { + if r.Filter.Prefix != "" { + return r.Filter.Prefix + } + return r.Filter.And.Prefix +} + +// Tags - a rule can either have tag under or under +// . This method returns all the tags from the +// rule in the format tag1=value1&tag2=value2 +func (r Rule) Tags() string { + if len(r.Filter.And.Tags) != 0 { + var buf bytes.Buffer + for _, t := range r.Filter.And.Tags { + if buf.Len() > 0 { + buf.WriteString("&") + } + buf.WriteString(t.String()) + } + return buf.String() + } + return "" +} + // Filter - a filter for a replication configuration Rule. type Filter struct { XMLName xml.Name `xml:"Filter" json:"-"` Prefix string `json:"Prefix,omitempty"` And And `xml:"And,omitempty" json:"And,omitempty"` + Tag Tag `xml:"Tag,omitempty" json:"Tag,omitempty"` +} + +// Validate - validates the filter element +func (f Filter) Validate() error { + // A Filter must have exactly one of Prefix, Tag, or And specified. + if !f.And.isEmpty() { + if f.Prefix != "" { + return errInvalidFilter + } + if !f.Tag.IsEmpty() { + return errInvalidFilter + } + } + if f.Prefix != "" { + if !f.Tag.IsEmpty() { + return errInvalidFilter + } + } + if !f.Tag.IsEmpty() { + if err := f.Tag.Validate(); err != nil { + return err + } + } + return nil } // Tag - a tag for a replication configuration Rule filter. @@ -57,6 +300,27 @@ type Tag struct { Value string `xml:"Value,omitempty" json:"Value,omitempty"` } +func (tag Tag) String() string { + return tag.Key + "=" + tag.Value +} + +// IsEmpty returns whether this tag is empty or not. +func (tag Tag) IsEmpty() bool { + return tag.Key == "" +} + +// Validate checks this tag. +func (tag Tag) Validate() error { + if len(tag.Key) == 0 || utf8.RuneCountInString(tag.Key) > 128 { + return fmt.Errorf("Invalid Tag Key") + } + + if utf8.RuneCountInString(tag.Value) > 256 { + return fmt.Errorf("Invalid Tag Value") + } + return nil +} + // Destination - destination in ReplicationConfiguration. type Destination struct { XMLName xml.Name `xml:"Destination" json:"-"` @@ -71,6 +335,11 @@ type And struct { Tags []Tag `xml:"Tag,omitempty" json:"Tags,omitempty"` } +// isEmpty returns true if Tags field is null +func (a And) isEmpty() bool { + return len(a.Tags) == 0 && a.Prefix == "" +} + // Status represents Enabled/Disabled status type Status string From 6e95dc5e46e15068c8e3048deb45055d9ecfa986 Mon Sep 17 00:00:00 2001 From: Minio Trusted Date: Tue, 4 Aug 2020 19:14:49 +0000 Subject: [PATCH 213/215] Update version to next release --- api.go | 2 +- go.mod | 2 ++ go.sum | 7 +++++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/api.go b/api.go index 09b1e0bb1..206e05dee 100644 --- a/api.go +++ b/api.go @@ -108,7 +108,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v7.0.3" + libraryVersion = "v7.0.4" ) // User Agent should always following the below style. diff --git a/go.mod b/go.mod index 448b4bbf1..670f6ea94 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/minio/minio-go/v7 go 1.12 require ( + github.com/dustin/go-humanize v1.0.0 // indirect github.com/google/uuid v1.1.1 github.com/json-iterator/go v1.1.10 github.com/klauspost/cpuid v1.3.1 // indirect @@ -13,6 +14,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect github.com/rs/xid v1.2.1 + github.com/sirupsen/logrus v1.6.0 // indirect github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect github.com/stretchr/testify v1.4.0 // indirect golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 diff --git a/go.sum b/go.sum index 1858b6ff3..1dce33dd4 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -14,6 +16,7 @@ github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -37,11 +40,14 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= @@ -57,6 +63,7 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From 210edc6e8bf9542930a043fa3eeb07463a7e67bd Mon Sep 17 00:00:00 2001 From: poornas Date: Thu, 6 Aug 2020 16:00:15 -0700 Subject: [PATCH 214/215] fix: change AddRule method and reuse RoleArn (#1360) --- docs/API.md | 7 +++---- examples/s3/setbucketreplication.go | 2 +- pkg/replication/replication.go | 26 +++++++++++++++++--------- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/docs/API.md b/docs/API.md index 5fb16a804..77ae8b0e2 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1901,7 +1901,7 @@ fmt.Printf("%+v\n", versioningConfig) ### SetBucketReplication(ctx context.Context, bucketname, cfg replication.Config) error -Set replication configuration on a bucket. To use this API with MinIO server, ReplicationArn should be set in the replication config. Replication ARN can be obtained by first defining the replication target on MinIO using `mc admin bucket replication set` to associate the source and destination buckets for replication with the replication endpoint. Next, issue a `mc admin bucket remote` to fetch the replication ARN associated with this replication endpoint. +Set replication configuration on a bucket. Role can be obtained by first defining the replication target on MinIO using `mc admin bucket remote set` to associate the source and destination buckets for replication with the replication endpoint. __Parameters__ @@ -1921,7 +1921,7 @@ __Example__ ```go replicationStr := ` - + Disabled @@ -1955,8 +1955,7 @@ replicationConfig := replication.Config{} if err := xml.Unmarshal([]byte(replicationStr), &replicationConfig); err != nil { log.Fatalln(err) } -// this is optional for replication with MinIO server. -cfg.ReplicationArn := "arn:minio:s3::598361bf-3cec-49a7-b529-ce870a34d759:*" +cfg.Role := "arn:minio:s3::598361bf-3cec-49a7-b529-ce870a34d759:*" err = minioClient.SetBucketReplication(context.Background(), "my-bucketname", replicationConfig) if err != nil { fmt.Println(err) diff --git a/examples/s3/setbucketreplication.go b/examples/s3/setbucketreplication.go index 62437cded..173e619e9 100644 --- a/examples/s3/setbucketreplication.go +++ b/examples/s3/setbucketreplication.go @@ -56,7 +56,7 @@ func main() { } // This replication ARN should have been generated for replication endpoint using `mc admin bucket remote` command - replCfg.ReplicationARN = "arn:minio:replica::dadddae7-f1d7-440f-b5d6-651aa9a8c8a7:dest" + replCfg.Role = "arn:minio:replica::dadddae7-f1d7-440f-b5d6-651aa9a8c8a7:dest" // Set replication config on a bucket err = s3Client.SetBucketReplication(context.Background(), "my-bucketname", replCfg) if err != nil { diff --git a/pkg/replication/replication.go b/pkg/replication/replication.go index 11ec67982..bb9581a99 100644 --- a/pkg/replication/replication.go +++ b/pkg/replication/replication.go @@ -20,6 +20,7 @@ import ( "bytes" "encoding/xml" "fmt" + "strconv" "strings" "unicode/utf8" @@ -46,7 +47,7 @@ type Options struct { ID string Prefix string Disable bool - Priority int + Priority string TagString string StorageClass string Arn string @@ -74,8 +75,7 @@ func (opts Options) Tags() []Tag { type Config struct { XMLName xml.Name `xml:"ReplicationConfiguration" json:"-"` Rules []Rule `xml:"Rule" json:"Rules"` - // ReplicationARN is a MinIO only extension and optional for AWS - ReplicationARN string `xml:"ReplicationArn,omitempty" json:"ReplicationArn,omitempty"` + Role string `xml:"Role" json:"Role"` } // Empty returns true if config is not set @@ -108,16 +108,24 @@ func (c *Config) AddRule(opts Options) error { if opts.Disable { status = Disabled } - tokens := strings.Split(opts.Arn, ":") + arnStr := opts.Arn + if opts.Arn == "" { + arnStr = c.Role + } + tokens := strings.Split(arnStr, ":") if len(tokens) != 6 { return fmt.Errorf("invalid format for replication Arn") } - if c.ReplicationARN == "" { // for new configurations - c.ReplicationARN = opts.Arn + if c.Role == "" { // for new configurations + c.Role = opts.Arn + } + priority, err := strconv.Atoi(opts.Priority) + if err != nil { + return err } newRule := Rule{ ID: opts.ID, - Priority: opts.Priority, + Priority: priority, Status: status, Filter: filter, Destination: Destination{ @@ -138,7 +146,7 @@ func (c *Config) AddRule(opts Options) error { if rule.ID != newRule.ID { continue } - if newRule.Status == Disabled && rule.ID == newRule.ID { + if opts.Priority == "" && rule.ID == newRule.ID { // inherit priority from existing rule, required field on server newRule.Priority = rule.Priority } @@ -197,7 +205,7 @@ func (r Rule) Validate() error { return err } - if r.Priority <= 0 && r.Status == Enabled { + if r.Priority < 0 && r.Status == Enabled { return fmt.Errorf("Priority must be set for the rule") } From f0460bf3dc247585c128b11196d47a49d49c2c7d Mon Sep 17 00:00:00 2001 From: poornas Date: Thu, 6 Aug 2020 17:00:57 -0700 Subject: [PATCH 215/215] Add SetOption to edit replication rule (#1362) Also adding a field to manage rule state --- pkg/replication/replication.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/pkg/replication/replication.go b/pkg/replication/replication.go index bb9581a99..57e63a08c 100644 --- a/pkg/replication/replication.go +++ b/pkg/replication/replication.go @@ -35,6 +35,9 @@ type OptionType string const ( // AddOption specifies addition of rule to config AddOption OptionType = "Add" + // SetOption specifies modification of existing rule to config + SetOption OptionType = "Set" + // RemoveOption specifies rule options are for removing a rule RemoveOption OptionType = "Remove" // ImportOption is for getting current config @@ -46,7 +49,7 @@ type Options struct { Op OptionType ID string Prefix string - Disable bool + RuleStatus string Priority string TagString string StorageClass string @@ -104,8 +107,12 @@ func (c *Config) AddRule(opts Options) error { if opts.ID == "" { opts.ID = xid.New().String() } - status := Enabled - if opts.Disable { + var status Status + // toggle rule status for edit option + switch opts.RuleStatus { + case "enable": + status = Enabled + case "disable": status = Disabled } arnStr := opts.Arn @@ -150,6 +157,9 @@ func (c *Config) AddRule(opts Options) error { // inherit priority from existing rule, required field on server newRule.Priority = rule.Priority } + if opts.RuleStatus == "" { + newRule.Status = rule.Status + } c.Rules[i] = newRule ruleFound = true break @@ -158,7 +168,9 @@ func (c *Config) AddRule(opts Options) error { if err := newRule.Validate(); err != nil { return err } - + if !ruleFound && opts.Op == SetOption { + return fmt.Errorf("Rule with ID %s not found in replication configuration", opts.ID) + } if !ruleFound { c.Rules = append(c.Rules, newRule) }