diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index 6ec122f..96bd6fb 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -28,7 +28,7 @@ jobs: --health-retries 5 # https://github.com/mosn/layotto/blob/main/docker/layotto-etcd/docker-compose.yml etcd: - image: bitnami/etcd + image: bitnamilegacy/etcd env: ALLOW_NONE_AUTHENTICATION: yes ETCD_ADVERTISE_CLIENT_URLS: http://0.0.0.0:2379 diff --git a/src/client/Oss.ts b/src/client/Oss.ts index a8ec7f4..8ed96a7 100644 --- a/src/client/Oss.ts +++ b/src/client/Oss.ts @@ -3,30 +3,88 @@ import { pipelinePromise } from '../utils'; import { CopyObjectRequest, DeleteObjectRequest, + DeleteObjectsRequest, + IsObjectExistRequest, GetObjectRequest, GetObjectResponse, HeadObjectRequest, ListObjectsRequest, + ListObjectVersionsRequest, PutObjectRequest, PutObjectResponse, SignUrlRequest, + PutObjectTaggingRequest, + DeleteObjectTaggingRequest, + GetObjectTaggingRequest, + GetObjectCannedAclRequest, + PutObjectCannedAclRequest, + CreateMultipartUploadRequest, + UploadPartRequest, + UploadPartCopyRequest, + CompleteMultipartUploadRequest, + AbortMultipartUploadRequest, + ListMultipartUploadsRequest, + ListPartsRequest, + AppendObjectRequest, + RestoreObjectRequest, + UpdateBandwidthRateLimitRequest, } from '../types/Oss'; import { + AbortMultipartUploadInput, + AbortMultipartUploadOutput, + AppendObjectInput, + AppendObjectOutput, + CompleteMultipartUploadInput, + CompleteMultipartUploadOutput, + CompletedMultipartUpload, + CompletedPart, CopyObjectInput, CopyObjectOutput, CopySource, + CreateMultipartUploadInput, + CreateMultipartUploadOutput, + Delete, DeleteObjectInput, DeleteObjectOutput, + DeleteObjectsInput, + DeleteObjectsOutput, + DeleteObjectTaggingInput, + DeleteObjectTaggingOutput, + GetObjectCannedAclInput, + GetObjectCannedAclOutput, GetObjectInput, GetObjectOutput, + GetObjectTaggingInput, + GetObjectTaggingOutput, HeadObjectInput, HeadObjectOutput, + IsObjectExistInput, + IsObjectExistOutput, + ListMultipartUploadsInput, + ListMultipartUploadsOutput, ListObjectsInput, ListObjectsOutput, + ListObjectVersionsInput, + ListObjectVersionsOutput, + ListPartsInput, + ListPartsOutput, + ObjectIdentifier, + PutObjectCannedAclInput, + PutObjectCannedAclOutput, PutObjectInput, PutObjectOutput, + PutObjectTaggingInput, + PutObjectTaggingOutput, + RestoreObjectInput, + RestoreObjectOutput, + RestoreRequest, SignURLInput, SignURLOutput, + UpdateBandwidthRateLimitInput, + UploadPartCopyInput, + UploadPartCopyOutput, + UploadPartInput, + UploadPartOutput, } from '../../proto/extension/v1/s3/oss_pb'; import { ObjectStorageServiceClient } from '../../proto/extension/v1/s3/oss_grpc_pb'; import { API, APIOptions } from './API'; @@ -150,8 +208,10 @@ export class Oss extends API { } } - private async* getObjectBufferIterator(firstChunk: GetObjectOutput, request: AsyncGenerator): AsyncGenerator { - yield firstChunk.getBody_asU8(); + private async* getObjectBufferIterator(firstChunk: GetObjectOutput | undefined, request: AsyncGenerator): AsyncGenerator { + if (firstChunk) { + yield firstChunk.getBody_asU8(); + } for await (const chunk of request) { yield chunk.getBody_asU8(); } @@ -232,7 +292,7 @@ export class Oss extends API { const firstChunk = (await getObjectIterator.next()).value; const getObjectBufIterator = this.getObjectBufferIterator(firstChunk, getObjectIterator); return { - ...firstChunk.toObject(), + ...firstChunk?.toObject(), object: PassThrough.from(getObjectBufIterator), }; @@ -404,4 +464,698 @@ export class Oss extends API { }); }); } + + async deleteObjects(request: DeleteObjectsRequest): Promise { + const req = new DeleteObjectsInput(); + req.setStoreName(request.storeName); + req.setBucket(request.bucket); + + const deleteObj = new Delete(); + const objectIdentifiers = request.objects.map(obj => { + const identifier = new ObjectIdentifier(); + identifier.setKey(this.#objectKey(obj.key)); + if (obj.versionId) { + identifier.setVersionId(obj.versionId); + } + return identifier; + }); + deleteObj.setObjectsList(objectIdentifiers); + if (request.quiet !== undefined) { + deleteObj.setQuiet(request.quiet); + } + req.setDelete(deleteObj); + + if (request.requestPayer) { + req.setRequestPayer(request.requestPayer); + } + + const metadata = this.createMetadata(request, this.options.defaultRequestMeta); + return new Promise((resolve, reject) => { + this.ossClient.deleteObjects(req, metadata, (err, response) => { + if (err) { + return reject(err); + } + return resolve(response!.toObject()); + }); + }); + } + + async isObjectExist(request: IsObjectExistRequest): Promise { + const key = this.#objectKey(request.key); + const req = new IsObjectExistInput(); + req.setStoreName(request.storeName); + req.setBucket(request.bucket); + req.setKey(key); + if (request.versionId) { + req.setVersionId(request.versionId); + } + + const metadata = this.createMetadata(request, this.options.defaultRequestMeta); + return new Promise((resolve, reject) => { + this.ossClient.isObjectExist(req, metadata, (err, response) => { + if (err) { + return reject(err); + } + return resolve(response!.toObject()); + }); + }); + } + + async listObjectVersions(request: ListObjectVersionsRequest): Promise { + const req = new ListObjectVersionsInput(); + req.setStoreName(request.storeName); + req.setBucket(request.bucket); + if (request.delimiter) { + req.setDelimiter(request.delimiter); + } + if (request.encodingType) { + req.setEncodingType(request.encodingType); + } + if (request.expectedBucketOwner) { + req.setExpectedBucketOwner(request.expectedBucketOwner); + } + if (request.keyMarker) { + req.setKeyMarker(request.keyMarker); + } + if (request.maxKeys) { + req.setMaxKeys(request.maxKeys); + } + if (request.prefix) { + req.setPrefix(request.prefix); + } + if (request.versionIdMarker) { + req.setVersionIdMarker(request.versionIdMarker); + } + + const metadata = this.createMetadata(request, this.options.defaultRequestMeta); + return new Promise((resolve, reject) => { + this.ossClient.listObjectVersions(req, metadata, (err, response) => { + if (err) { + return reject(err); + } + return resolve(response!.toObject()); + }); + }); + } + + async putObjectTagging(request: PutObjectTaggingRequest): Promise { + const key = this.#objectKey(request.key); + const req = new PutObjectTaggingInput(); + req.setStoreName(request.storeName); + req.setBucket(request.bucket); + req.setKey(key); + + const tagsMap = req.getTagsMap(); + Object.entries(request.tags).forEach(([ k, v ]) => { + tagsMap.set(k, v); + }); + + if (request.versionId) { + req.setVersionId(request.versionId); + } + + const metadata = this.createMetadata(request, this.options.defaultRequestMeta); + return new Promise((resolve, reject) => { + this.ossClient.putObjectTagging(req, metadata, (err, response) => { + if (err) { + return reject(err); + } + return resolve(response!.toObject()); + }); + }); + } + + async deleteObjectTagging(request: DeleteObjectTaggingRequest): Promise { + const key = this.#objectKey(request.key); + const req = new DeleteObjectTaggingInput(); + req.setStoreName(request.storeName); + req.setBucket(request.bucket); + req.setKey(key); + if (request.versionId) { + req.setVersionId(request.versionId); + } + if (request.expectedBucketOwner) { + req.setExpectedBucketOwner(request.expectedBucketOwner); + } + + const metadata = this.createMetadata(request, this.options.defaultRequestMeta); + return new Promise((resolve, reject) => { + this.ossClient.deleteObjectTagging(req, metadata, (err, response) => { + if (err) { + return reject(err); + } + return resolve(response!.toObject()); + }); + }); + } + + async getObjectTagging(request: GetObjectTaggingRequest): Promise { + const key = this.#objectKey(request.key); + const req = new GetObjectTaggingInput(); + req.setStoreName(request.storeName); + req.setBucket(request.bucket); + req.setKey(key); + if (request.versionId) { + req.setVersionId(request.versionId); + } + if (request.expectedBucketOwner) { + req.setExpectedBucketOwner(request.expectedBucketOwner); + } + if (request.requestPayer) { + req.setRequestPayer(request.requestPayer); + } + + const metadata = this.createMetadata(request, this.options.defaultRequestMeta); + return new Promise((resolve, reject) => { + this.ossClient.getObjectTagging(req, metadata, (err, response) => { + if (err) { + return reject(err); + } + return resolve(response!.toObject()); + }); + }); + } + + async getObjectCannedAcl(request: GetObjectCannedAclRequest): Promise { + const key = this.#objectKey(request.key); + const req = new GetObjectCannedAclInput(); + req.setStoreName(request.storeName); + req.setBucket(request.bucket); + req.setKey(key); + if (request.versionId) { + req.setVersionId(request.versionId); + } + + const metadata = this.createMetadata(request, this.options.defaultRequestMeta); + return new Promise((resolve, reject) => { + this.ossClient.getObjectCannedAcl(req, metadata, (err, response) => { + if (err) { + return reject(err); + } + return resolve(response!.toObject()); + }); + }); + } + + async putObjectCannedAcl(request: PutObjectCannedAclRequest): Promise { + const key = this.#objectKey(request.key); + const req = new PutObjectCannedAclInput(); + req.setStoreName(request.storeName); + req.setBucket(request.bucket); + req.setKey(key); + req.setAcl(request.acl); + if (request.versionId) { + req.setVersionId(request.versionId); + } + + const metadata = this.createMetadata(request, this.options.defaultRequestMeta); + return new Promise((resolve, reject) => { + this.ossClient.putObjectCannedAcl(req, metadata, (err, response) => { + if (err) { + return reject(err); + } + return resolve(response!.toObject()); + }); + }); + } + + async createMultipartUpload(request: CreateMultipartUploadRequest): Promise { + const key = this.#objectKey(request.key); + const req = new CreateMultipartUploadInput(); + req.setStoreName(request.storeName); + req.setBucket(request.bucket); + req.setKey(key); + if (request.acl) { + req.setAcl(request.acl); + } + if (request.bucketKeyEnabled) { + req.setBucketKeyEnabled(request.bucketKeyEnabled); + } + if (request.cacheControl) { + req.setCacheControl(request.cacheControl); + } + if (request.contentDisposition) { + req.setContentDisposition(request.contentDisposition); + } + if (request.contentEncoding) { + req.setContentEncoding(request.contentEncoding); + } + if (request.contentLanguage) { + req.setContentLanguage(request.contentLanguage); + } + if (request.contentType) { + req.setContentType(request.contentType); + } + if (request.expectedBucketOwner) { + req.setExpectedBucketOwner(request.expectedBucketOwner); + } + if (request.expires) { + req.setExpires(request.expires); + } + if (request.serverSideEncryption) { + req.setServerSideEncryption(request.serverSideEncryption); + } + if (request.storageClass) { + req.setStorageClass(request.storageClass); + } + + const metadata = this.createMetadata(request, this.options.defaultRequestMeta); + return new Promise((resolve, reject) => { + this.ossClient.createMultipartUpload(req, metadata, (err, response) => { + if (err) { + return reject(err); + } + return resolve(response!.toObject()); + }); + }); + } + + private async* uploadPartIterator(request: UploadPartRequest): AsyncGenerator { + const key = this.#objectKey(request.key); + let hasChunk = false; + for await (const chunk of request.body) { + hasChunk = true; + const req = new UploadPartInput(); + req.setStoreName(request.storeName); + req.setBucket(request.bucket); + req.setKey(key); + req.setContentLength(request.contentLength); + req.setPartNumber(request.partNumber); + req.setUploadId(request.uploadId); + if (request.contentMd5) { + req.setContentMd5(request.contentMd5); + } + if (request.expectedBucketOwner) { + req.setExpectedBucketOwner(request.expectedBucketOwner); + } + if (request.requestPayer) { + req.setRequestPayer(request.requestPayer); + } + if (request.sseCustomerAlgorithm) { + req.setSseCustomerAlgorithm(request.sseCustomerAlgorithm); + } + if (request.sseCustomerKey) { + req.setSseCustomerKey(request.sseCustomerKey); + } + if (request.sseCustomerKeyMd5) { + req.setSseCustomerKeyMd5(request.sseCustomerKeyMd5); + } + req.setBody(chunk); + yield req; + } + + if (!hasChunk) { + // upload empty part + const req = new UploadPartInput(); + req.setStoreName(request.storeName); + req.setBucket(request.bucket); + req.setKey(key); + req.setContentLength(request.contentLength); + req.setPartNumber(request.partNumber); + req.setUploadId(request.uploadId); + if (request.contentMd5) { + req.setContentMd5(request.contentMd5); + } + if (request.expectedBucketOwner) { + req.setExpectedBucketOwner(request.expectedBucketOwner); + } + if (request.requestPayer) { + req.setRequestPayer(request.requestPayer); + } + if (request.sseCustomerAlgorithm) { + req.setSseCustomerAlgorithm(request.sseCustomerAlgorithm); + } + if (request.sseCustomerKey) { + req.setSseCustomerKey(request.sseCustomerKey); + } + if (request.sseCustomerKeyMd5) { + req.setSseCustomerKeyMd5(request.sseCustomerKeyMd5); + } + yield req; + } + } + + async uploadPart(request: UploadPartRequest): Promise { + let resolve; + let reject; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + const metadata = this.createMetadata(request, this.options.defaultRequestMeta); + const writeStream = this.ossClient.uploadPart(metadata, (err, res) => { + if (err) { + return reject(err); + } + return resolve(res.toObject()); + }); + const uploadPartIterator = this.uploadPartIterator(request); + await pipelinePromise, Writable>(uploadPartIterator, writeStream); + + return promise; + } + + async uploadPartCopy(request: UploadPartCopyRequest): Promise { + const key = this.#objectKey(request.key); + const req = new UploadPartCopyInput(); + req.setStoreName(request.storeName); + req.setBucket(request.bucket); + req.setKey(key); + req.setPartNumber(request.partNumber); + req.setUploadId(request.uploadId); + + const copySource = new CopySource(); + copySource.setCopySourceBucket(request.copySource.copySourceBucket); + copySource.setCopySourceKey(request.copySource.copySourceKey); + if (request.copySource.copySourceVersionId) { + copySource.setCopySourceVersionId(request.copySource.copySourceVersionId); + } + req.setCopySource(copySource); + + if (request.startPosition !== undefined) { + req.setStartPosition(request.startPosition); + } + if (request.partSize !== undefined) { + req.setPartSize(request.partSize); + } + + const metadata = this.createMetadata(request, this.options.defaultRequestMeta); + return new Promise((resolve, reject) => { + this.ossClient.uploadPartCopy(req, metadata, (err, response) => { + if (err) { + return reject(err); + } + return resolve(response!.toObject()); + }); + }); + } + + async completeMultipartUpload(request: CompleteMultipartUploadRequest): Promise { + const key = this.#objectKey(request.key); + const req = new CompleteMultipartUploadInput(); + req.setStoreName(request.storeName); + req.setBucket(request.bucket); + req.setKey(key); + req.setUploadId(request.uploadId); + + const multipartUpload = new CompletedMultipartUpload(); + const parts = request.parts.map(part => { + const completedPart = new CompletedPart(); + completedPart.setEtag(part.etag); + completedPart.setPartNumber(part.partNumber); + return completedPart; + }); + multipartUpload.setPartsList(parts); + req.setMultipartUpload(multipartUpload); + + if (request.requestPayer) { + req.setRequestPayer(request.requestPayer); + } + if (request.expectedBucketOwner) { + req.setExpectedBucketOwner(request.expectedBucketOwner); + } + + const metadata = this.createMetadata(request, this.options.defaultRequestMeta); + return new Promise((resolve, reject) => { + this.ossClient.completeMultipartUpload(req, metadata, (err, response) => { + if (err) { + return reject(err); + } + return resolve(response!.toObject()); + }); + }); + } + + async abortMultipartUpload(request: AbortMultipartUploadRequest): Promise { + const key = this.#objectKey(request.key); + const req = new AbortMultipartUploadInput(); + req.setStoreName(request.storeName); + req.setBucket(request.bucket); + req.setKey(key); + req.setUploadId(request.uploadId); + if (request.expectedBucketOwner) { + req.setExpectedBucketOwner(request.expectedBucketOwner); + } + if (request.requestPayer) { + req.setRequestPayer(request.requestPayer); + } + + const metadata = this.createMetadata(request, this.options.defaultRequestMeta); + return new Promise((resolve, reject) => { + this.ossClient.abortMultipartUpload(req, metadata, (err, response) => { + if (err) { + return reject(err); + } + return resolve(response!.toObject()); + }); + }); + } + + async listMultipartUploads(request: ListMultipartUploadsRequest): Promise { + const req = new ListMultipartUploadsInput(); + req.setStoreName(request.storeName); + req.setBucket(request.bucket); + if (request.delimiter) { + req.setDelimiter(request.delimiter); + } + if (request.encodingType) { + req.setEncodingType(request.encodingType); + } + if (request.expectedBucketOwner) { + req.setExpectedBucketOwner(request.expectedBucketOwner); + } + if (request.keyMarker) { + req.setKeyMarker(request.keyMarker); + } + if (request.maxUploads) { + req.setMaxUploads(request.maxUploads); + } + if (request.prefix) { + req.setPrefix(request.prefix); + } + if (request.uploadIdMarker) { + req.setUploadIdMarker(request.uploadIdMarker); + } + + const metadata = this.createMetadata(request, this.options.defaultRequestMeta); + return new Promise((resolve, reject) => { + this.ossClient.listMultipartUploads(req, metadata, (err, response) => { + if (err) { + return reject(err); + } + return resolve(response!.toObject()); + }); + }); + } + + async listParts(request: ListPartsRequest): Promise { + const key = this.#objectKey(request.key); + const req = new ListPartsInput(); + req.setStoreName(request.storeName); + req.setBucket(request.bucket); + req.setKey(key); + req.setUploadId(request.uploadId); + if (request.expectedBucketOwner) { + req.setExpectedBucketOwner(request.expectedBucketOwner); + } + if (request.maxParts) { + req.setMaxParts(request.maxParts); + } + if (request.partNumberMarker) { + req.setPartNumberMarker(request.partNumberMarker); + } + if (request.requestPayer) { + req.setRequestPayer(request.requestPayer); + } + + const metadata = this.createMetadata(request, this.options.defaultRequestMeta); + return new Promise((resolve, reject) => { + this.ossClient.listParts(req, metadata, (err, response) => { + if (err) { + return reject(err); + } + return resolve(response!.toObject()); + }); + }); + } + + private async* appendObjectIterator(request: AppendObjectRequest): AsyncGenerator { + const key = this.#objectKey(request.key); + let hasChunk = false; + for await (const chunk of request.body) { + hasChunk = true; + const req = new AppendObjectInput(); + req.setStoreName(request.storeName); + req.setBucket(request.bucket); + req.setKey(key); + if (request.position) { + req.setPosition(request.position); + } + if (request.acl) { + req.setAcl(request.acl); + } + if (request.cacheControl) { + req.setCacheControl(request.cacheControl); + } + if (request.contentDisposition) { + req.setContentDisposition(request.contentDisposition); + } + if (request.contentEncoding) { + req.setContentEncoding(request.contentEncoding); + } + if (request.contentMd5) { + req.setContentMd5(request.contentMd5); + } + if (request.expires) { + req.setExpires(request.expires); + } + if (request.storageClass) { + req.setStorageClass(request.storageClass); + } + if (request.serverSideEncryption) { + req.setServerSideEncryption(request.serverSideEncryption); + } + if (request.meta) { + req.setMeta(request.meta); + } + if (request.tags) { + const tagsMap = req.getTagsMap(); + Object.entries(request.tags).forEach(([ k, v ]) => { + tagsMap.set(k, v); + }); + } + req.setBody(chunk); + yield req; + } + + if (!hasChunk) { + // append empty content + const req = new AppendObjectInput(); + req.setStoreName(request.storeName); + req.setBucket(request.bucket); + req.setKey(key); + if (request.position) { + req.setPosition(request.position); + } + if (request.acl) { + req.setAcl(request.acl); + } + if (request.cacheControl) { + req.setCacheControl(request.cacheControl); + } + if (request.contentDisposition) { + req.setContentDisposition(request.contentDisposition); + } + if (request.contentEncoding) { + req.setContentEncoding(request.contentEncoding); + } + if (request.contentMd5) { + req.setContentMd5(request.contentMd5); + } + if (request.expires) { + req.setExpires(request.expires); + } + if (request.storageClass) { + req.setStorageClass(request.storageClass); + } + if (request.serverSideEncryption) { + req.setServerSideEncryption(request.serverSideEncryption); + } + if (request.meta) { + req.setMeta(request.meta); + } + if (request.tags) { + const tagsMap = req.getTagsMap(); + Object.entries(request.tags).forEach(([ k, v ]) => { + tagsMap.set(k, v); + }); + } + yield req; + } + } + + async appendObject(request: AppendObjectRequest): Promise { + let resolve; + let reject; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + const metadata = this.createMetadata(request, this.options.defaultRequestMeta); + const writeStream = this.ossClient.appendObject(metadata, (err, res) => { + if (err) { + return reject(err); + } + return resolve(res.toObject()); + }); + const appendObjectIterator = this.appendObjectIterator(request); + await pipelinePromise, Writable>(appendObjectIterator, writeStream); + + return promise; + } + + async restoreObject(request: RestoreObjectRequest): Promise { + const key = this.#objectKey(request.key); + const req = new RestoreObjectInput(); + req.setStoreName(request.storeName); + req.setBucket(request.bucket); + req.setKey(key); + + if (request.days !== undefined || request.tier !== undefined) { + const restoreRequest = new RestoreRequest(); + if (request.days !== undefined) { + restoreRequest.setDays(request.days); + } + if (request.tier) { + restoreRequest.setTier(request.tier); + } + req.setRestoreRequest(restoreRequest); + } + + if (request.versionId) { + req.setVersionId(request.versionId); + } + + const metadata = this.createMetadata(request, this.options.defaultRequestMeta); + return new Promise((resolve, reject) => { + this.ossClient.restoreObject(req, metadata, (err, response) => { + if (err) { + return reject(err); + } + return resolve(response!.toObject()); + }); + }); + } + + async updateDownloadBandwidthRateLimit(request: UpdateBandwidthRateLimitRequest): Promise { + const req = new UpdateBandwidthRateLimitInput(); + req.setStoreName(request.storeName); + req.setAverageRateLimitInBitsPerSec(request.averageRateLimitInBitsPerSec); + + const metadata = this.createMetadata(request, this.options.defaultRequestMeta); + return new Promise((resolve, reject) => { + this.ossClient.updateDownloadBandwidthRateLimit(req, metadata, err => { + if (err) { + return reject(err); + } + return resolve(); + }); + }); + } + + async updateUploadBandwidthRateLimit(request: UpdateBandwidthRateLimitRequest): Promise { + const req = new UpdateBandwidthRateLimitInput(); + req.setStoreName(request.storeName); + req.setAverageRateLimitInBitsPerSec(request.averageRateLimitInBitsPerSec); + + const metadata = this.createMetadata(request, this.options.defaultRequestMeta); + return new Promise((resolve, reject) => { + this.ossClient.updateUploadBandwidthRateLimit(req, metadata, err => { + if (err) { + return reject(err); + } + return resolve(); + }); + }); + } } diff --git a/src/client/State.ts b/src/client/State.ts index 91e6c67..32d1b15 100644 --- a/src/client/State.ts +++ b/src/client/State.ts @@ -37,7 +37,7 @@ import { SaveStateRequest, StateItem, } from '../types/State'; -import { isEmptyPBMessage, convertMapToKVString } from '../utils'; +import { convertMapToKVString } from '../utils'; export class State extends RuntimeAPI { // Saves an array of state objects @@ -69,12 +69,14 @@ export class State extends RuntimeAPI { return new Promise((resolve, reject) => { this.runtime.getState(req, this.createMetadata(request), (err, res: GetStateResponsePB) => { if (err) return reject(err); - if (isEmptyPBMessage(res)) { - return resolve(null); - } + const value = res.getData_asU8(); + if ( + value.length === 0 + && res.getEtag() === '' + ) return resolve(null); resolve({ key: request.key, - value: res.getData_asU8(), + value, etag: res.getEtag(), metadata: convertMapToKVString(res.getMetadataMap()), }); diff --git a/src/types/Oss.ts b/src/types/Oss.ts index e9be4c4..25b6302 100644 --- a/src/types/Oss.ts +++ b/src/types/Oss.ts @@ -120,3 +120,198 @@ export type SignUrlRequest = RequestWithMeta<{ method: string; expiredInSec: number; }>; + +export interface ObjectIdentifier { + key: string; + versionId?: string; +} + +export type DeleteObjectsRequest = RequestWithMeta<{ + storeName: string; + bucket: string; + objects: ObjectIdentifier[]; + quiet?: boolean; + requestPayer?: string; +}>; + +export type IsObjectExistRequest = RequestWithMeta<{ + storeName: string; + bucket: string; + key: string; + versionId?: string; +}>; + +export type PutObjectTaggingRequest = RequestWithMeta<{ + storeName: string; + bucket: string; + key: string; + tags: Record; + versionId?: string; +}>; + +export type DeleteObjectTaggingRequest = RequestWithMeta<{ + storeName: string; + bucket: string; + key: string; + versionId?: string; + expectedBucketOwner?: string; +}>; + +export type GetObjectTaggingRequest = RequestWithMeta<{ + storeName: string; + bucket: string; + key: string; + versionId?: string; + expectedBucketOwner?: string; + requestPayer?: string; +}>; + +export type GetObjectCannedAclRequest = RequestWithMeta<{ + storeName: string; + bucket: string; + key: string; + versionId?: string; +}>; + +export type PutObjectCannedAclRequest = RequestWithMeta<{ + storeName: string; + bucket: string; + key: string; + acl: string; + versionId?: string; +}>; + +export type CreateMultipartUploadRequest = RequestWithMeta<{ + storeName: string; + bucket: string; + key: string; + acl?: string; + bucketKeyEnabled?: boolean; + cacheControl?: string; + contentDisposition?: string; + contentEncoding?: string; + contentLanguage?: string; + contentType?: string; + expectedBucketOwner?: string; + expires?: number; + serverSideEncryption?: string; + storageClass?: string; +}>; + +export type UploadPartRequest = RequestWithMeta<{ + storeName: string; + bucket: string; + key: string; + body: Readable; + contentLength: number; + partNumber: number; + uploadId: string; + contentMd5?: string; + expectedBucketOwner?: string; + requestPayer?: string; + sseCustomerAlgorithm?: string; + sseCustomerKey?: string; + sseCustomerKeyMd5?: string; +}>; + +export interface CompletedPart { + etag: string; + partNumber: number; +} + +export type CompleteMultipartUploadRequest = RequestWithMeta<{ + storeName: string; + bucket: string; + key: string; + uploadId: string; + parts: CompletedPart[]; + requestPayer?: string; + expectedBucketOwner?: string; +}>; + +export type AbortMultipartUploadRequest = RequestWithMeta<{ + storeName: string; + bucket: string; + key: string; + uploadId: string; + expectedBucketOwner?: string; + requestPayer?: string; +}>; + +export type ListMultipartUploadsRequest = RequestWithMeta<{ + storeName: string; + bucket: string; + delimiter?: string; + encodingType?: string; + expectedBucketOwner?: string; + keyMarker?: string; + maxUploads?: number; + prefix?: string; + uploadIdMarker?: string; +}>; + +export type ListPartsRequest = RequestWithMeta<{ + storeName: string; + bucket: string; + key: string; + uploadId: string; + expectedBucketOwner?: string; + maxParts?: number; + partNumberMarker?: number; + requestPayer?: string; +}>; + +export type UploadPartCopyRequest = RequestWithMeta<{ + storeName: string; + bucket: string; + key: string; + copySource: CopySource; + partNumber: number; + uploadId: string; + startPosition?: number; + partSize?: number; +}>; + +export type ListObjectVersionsRequest = RequestWithMeta<{ + storeName: string; + bucket: string; + delimiter?: string; + encodingType?: string; + expectedBucketOwner?: string; + keyMarker?: string; + maxKeys?: number; + prefix?: string; + versionIdMarker?: string; +}>; + +export type AppendObjectRequest = RequestWithMeta<{ + storeName: string; + bucket: string; + key: string; + body: Readable; + position?: number; + acl?: string; + cacheControl?: string; + contentDisposition?: string; + contentEncoding?: string; + contentMd5?: string; + expires?: number; + storageClass?: string; + serverSideEncryption?: string; + meta?: string; + tags?: Record; +}>; + +export type RestoreObjectRequest = RequestWithMeta<{ + storeName: string; + bucket: string; + key: string; + days?: number; + tier?: string; + versionId?: string; +}>; + +export type UpdateBandwidthRateLimitRequest = RequestWithMeta<{ + storeName: string; + averageRateLimitInBitsPerSec: number; +}>; diff --git a/test/unit/client/Oss.test.ts b/test/unit/client/Oss.test.ts index 79d9d87..556ba4d 100644 --- a/test/unit/client/Oss.test.ts +++ b/test/unit/client/Oss.test.ts @@ -4,9 +4,11 @@ import { createReadStream } from 'node:fs'; import path from 'node:path'; import crypto from 'node:crypto'; import { Client } from '../../../src'; +import { randomUUID } from 'node:crypto'; describe.skip('client/Oss.test.ts', () => { - const client = new Client('34901', '127.0.0.1', { ossEnable: true }); + const client = new Client('34904', '127.0.0.1', { ossEnable: true }); + it('test put object', async () => { const hello = await client.oss.put({ storeName: 'oss_demo', @@ -158,4 +160,367 @@ describe.skip('client/Oss.test.ts', () => { }); assert(res); }); + + it('test deleteObjects - batch delete', async () => { + // Create test objects + await client.oss.put({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key: 'test_batch_delete_1.txt', + body: Readable.from(Buffer.from('test1')), + contentLength: 5, + }); + await client.oss.put({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key: 'test_batch_delete_2.txt', + body: Readable.from(Buffer.from('test2')), + contentLength: 5, + }); + + // Delete multiple objects + const deleteRes = await client.oss.deleteObjects({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + objects: [ + { key: 'test_batch_delete_1.txt' }, + { key: 'test_batch_delete_2.txt' }, + ], + }); + assert(deleteRes); + assert(deleteRes.deletedList); + }); + + it('test isObjectExist', async () => { + await client.oss.put({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key: 'test_exist.txt', + body: Readable.from(Buffer.from('exist')), + contentLength: 5, + }); + + // Check existing object + const existRes = await client.oss.isObjectExist({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key: 'test_exist.txt', + }); + assert.equal(existRes.fileExist, true); + + // Check non-existing object + const notExistRes = await client.oss.isObjectExist({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key: 'test_not_exist.txt', + }); + assert.equal(notExistRes.fileExist, false); + }); + + it('test object tagging', async () => { + await client.oss.put({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key: 'test_tagging.txt', + body: Readable.from(Buffer.from('test tagging')), + contentLength: 12, + }); + + // Put tags + const putTagRes = await client.oss.putObjectTagging({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key: 'test_tagging.txt', + tags: { + env: 'test', + team: 'dev', + }, + }); + assert(putTagRes); + + // Get tags + const getTagRes = await client.oss.getObjectTagging({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key: 'test_tagging.txt', + }); + assert(getTagRes.tagsMap); + assert(getTagRes.tagsMap.length > 0); + + // Delete tags + const deleteTagRes = await client.oss.deleteObjectTagging({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key: 'test_tagging.txt', + }); + assert(deleteTagRes); + }); + + it('test object ACL', async () => { + await client.oss.put({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key: 'test_acl.txt', + body: Readable.from(Buffer.from('test acl')), + contentLength: 8, + }); + + // Put ACL + const putAclRes = await client.oss.putObjectCannedAcl({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key: 'test_acl.txt', + acl: 'private', + }); + assert(putAclRes); + + // Get ACL + const getAclRes = await client.oss.getObjectCannedAcl({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key: 'test_acl.txt', + }); + assert(getAclRes); + assert(getAclRes.cannedAcl); + }); + + it('test multipart upload', async () => { + // Create multipart upload + const createRes = await client.oss.createMultipartUpload({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key: 'test_multipart.txt', + }); + assert(createRes.uploadId); + const uploadId = createRes.uploadId; + + // Upload part 1 + const part1Data = Buffer.alloc(1024 * 1024).fill('a'); // 1MB + const uploadPart1Res = await client.oss.uploadPart({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key: 'test_multipart.txt', + uploadId, + partNumber: 1, + body: Readable.from(part1Data), + contentLength: part1Data.length, + }); + assert(uploadPart1Res.etag); + + // Upload part 2 + const part2Data = Buffer.alloc(1024 * 1024).fill('b'); // 1MB + const uploadPart2Res = await client.oss.uploadPart({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key: 'test_multipart.txt', + uploadId, + partNumber: 2, + body: Readable.from(part2Data), + contentLength: part2Data.length, + }); + assert(uploadPart2Res.etag); + + // List parts + const listPartsRes = await client.oss.listParts({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key: 'test_multipart.txt', + uploadId, + }); + assert(listPartsRes.partsList); + assert.equal(listPartsRes.partsList.length, 2); + + // Complete multipart upload + const completeRes = await client.oss.completeMultipartUpload({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key: 'test_multipart.txt', + uploadId, + parts: [ + { etag: uploadPart1Res.etag, partNumber: 1 }, + { etag: uploadPart2Res.etag, partNumber: 2 }, + ], + }); + assert(completeRes); + assert(completeRes.etag); + }); + + it('test abort multipart upload', async () => { + // Create multipart upload + const createRes = await client.oss.createMultipartUpload({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key: 'test_abort_multipart.txt', + }); + assert(createRes.uploadId); + const uploadId = createRes.uploadId; + + // Upload a part + const partData = Buffer.alloc(1024 * 1024).fill('x'); + await client.oss.uploadPart({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key: 'test_abort_multipart.txt', + uploadId, + partNumber: 1, + body: Readable.from(partData), + contentLength: partData.length, + }); + + // Abort multipart upload + const abortRes = await client.oss.abortMultipartUpload({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key: 'test_abort_multipart.txt', + uploadId, + }); + assert(abortRes); + }); + + it('test listMultipartUploads', async () => { + // Create a multipart upload + await client.oss.createMultipartUpload({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key: 'test_list_multipart.txt', + }); + + // List multipart uploads + const listRes = await client.oss.listMultipartUploads({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + prefix: 'test_list_', + }); + assert(listRes); + }); + + it('test appendObject', async () => { + const key = `test_append${randomUUID()}.txt`; + // First append + const append1Res = await client.oss.appendObject({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key, + body: Readable.from(Buffer.from('hello ')), + // position: 0, + }); + assert(append1Res); + const nextPosition = append1Res.appendPosition; + + // Second append + const append2Res = await client.oss.appendObject({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key, + body: Readable.from(Buffer.from('world')), + position: nextPosition, + }); + assert(append2Res); + + // Verify the content + const res = await client.oss.get({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key, + }); + const buf: Uint8Array[] = []; + for await (const chunk of res.object) { + buf.push(chunk); + } + const data = Buffer.concat(buf).toString(); + assert.equal(data, 'hello world'); + }); + + it('test uploadPartCopy', async () => { + const key = `test_part_copy_source${randomUUID()}.txt`; + // Create source object + const sourceData = Buffer.alloc(2 * 1024 * 1024).fill('s'); // 5MB + await client.oss.put({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key, + body: Readable.from(sourceData), + contentLength: sourceData.length, + }); + + // Create multipart upload for destination + const createRes = await client.oss.createMultipartUpload({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key, + }); + const uploadId = createRes.uploadId; + + // Copy part from source + const copyPartRes = await client.oss.uploadPartCopy({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key, + uploadId, + partNumber: 1, + copySource: { + copySourceBucket: 'antsys-tnpmbuild', + copySourceKey: key, + }, + }); + assert(copyPartRes); + assert(copyPartRes.copyPartResult?.etag); + + // Complete multipart upload + await client.oss.completeMultipartUpload({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key, + uploadId, + parts: [ + { etag: copyPartRes.copyPartResult!.etag, partNumber: 1 }, + ], + }); + }); + + it('test head object', async () => { + await client.oss.put({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key: 'test_head.txt', + body: Readable.from(Buffer.from('test head')), + contentLength: 9, + }); + + const headRes = await client.oss.head({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key: 'test_head.txt', + }); + assert(headRes); + console.log(headRes); + // assert(headRes.contentLength); + // assert.equal(headRes.contentLength, 9); + }); + + it('test signUrl', async () => { + const signRes = await client.oss.signUrl({ + storeName: 'oss_demo', + bucket: 'antsys-tnpmbuild', + key: 'test_sign.txt', + method: 'GET', + expiredInSec: 3600, + }); + assert(signRes); + assert(signRes.signedUrl); + }); + + it('test updateBandwidthRateLimit', async () => { + // Update download bandwidth rate limit + await client.oss.updateDownloadBandwidthRateLimit({ + storeName: 'oss_demo', + averageRateLimitInBitsPerSec: 1024 * 1024 * 8, // 1MB/s + }); + + // Update upload bandwidth rate limit + await client.oss.updateUploadBandwidthRateLimit({ + storeName: 'oss_demo', + averageRateLimitInBitsPerSec: 1024 * 1024 * 8, // 1MB/s + }); + }); });