diff --git a/.github/workflows/e2e-blob.yml b/.github/workflows/e2e-blob.yml new file mode 100644 index 00000000..7750e7d7 --- /dev/null +++ b/.github/workflows/e2e-blob.yml @@ -0,0 +1,67 @@ +name: E2E Blob Storage + +on: + workflow_dispatch: + schedule: + - cron: "0 8 * * 0" # Sundays at 08:00 UTC + +permissions: + contents: read + +jobs: + e2e-blob: + runs-on: ubuntu-latest + name: E2E blob upload/download + + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 22 + cache: "npm" + + - name: Install npm packages + run: | + npm ci --ignore-scripts + npm run ci:postinstall + + - name: Run E2E blob storage tests + env: + MOPS_TEST_E2E: "1" + MOPS_NETWORK: staging + run: | + cd cli && NODE_OPTIONS="--experimental-vm-modules" \ + npx jest tests/e2e-blob-storage.test.ts --testTimeout 180000 + + e2e-blob-publish: + runs-on: ubuntu-latest + name: E2E blob publish + install + if: github.event_name == 'workflow_dispatch' + + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 22 + cache: "npm" + + - name: Install npm packages + run: | + npm ci --ignore-scripts + npm run ci:postinstall + + - name: Write identity PEM + env: + MOPS_IDENTITY_PEM: ${{ secrets.MOPS_IDENTITY_PEM }} + run: | + mkdir -p ~/.config/mops + echo "$MOPS_IDENTITY_PEM" > ~/.config/mops/identity.pem + + - name: Run E2E publish + install tests + env: + MOPS_TEST_E2E: "1" + MOPS_IDENTITY_PEM: ${{ secrets.MOPS_IDENTITY_PEM }} + MOPS_NETWORK: staging + run: | + cd cli && NODE_OPTIONS="--experimental-vm-modules" \ + npx jest tests/e2e-blob-storage.test.ts --testTimeout 180000 diff --git a/backend/main/PackagePublisher.mo b/backend/main/PackagePublisher.mo index 9447768d..2d142281 100644 --- a/backend/main/PackagePublisher.mo +++ b/backend/main/PackagePublisher.mo @@ -42,6 +42,23 @@ module { path : Text; }; + type PackageId = Types.PackageId; + + func _isValidBlobHash(hash : Text) : Bool { + if (Text.size(hash) != 71) return false; + if (not Text.startsWith(hash, #text("sha256:"))) return false; + let hexPart = switch (Text.stripStart(hash, #text("sha256:"))) { + case null return false; + case (?h) h; + }; + for (c in hexPart.chars()) { + if (not ((c >= '0' and c <= '9') or (c >= 'a' and c <= 'f'))) { + return false; + }; + }; + true; + }; + public class PackagePublisher(registry : Registry.Registry, storageManager : StorageManager.StorageManager) { let MAX_PACKAGE_FILES = 1000; let MAX_PACKAGE_SIZE = 1024 * 1024 * 28; // 28MB @@ -55,12 +72,11 @@ module { let publishingBenchmarks = TrieMap.TrieMap(Text.equal, Text.hash); let publishingDocsCoverage = TrieMap.TrieMap(Text.equal, Text.hash); - public func startPublish(caller : Principal, config : PackageConfigV3) : async Result.Result { + func _validatePublishConfig(caller : Principal, config : PackageConfigV3) : Result.Result<(), PublishingErr> { if (Principal.isAnonymous(caller)) { return #err("Unauthorized"); }; - // validate config switch (validateConfig(config)) { case (#ok) {}; case (#err(err)) { @@ -70,12 +86,10 @@ module { let isNewPackage = registry.getHighestVersion(config.name) == null; - // check permissions if (not isNewPackage and not registry.isOwner(config.name, caller) and not registry.isMaintainer(config.name, caller)) { return #err("Only owners and maintainers can publish packages"); }; - // deny '.' and '_' in name for new packages if (isNewPackage) { for (char in config.name.chars()) { let err = #err("invalid config: unexpected char '" # Char.toText(char) # "' in name '" # config.name # "'"); @@ -85,7 +99,6 @@ module { }; }; - // check if the same version is published switch (registry.getPackageVersions(config.name)) { case (?versions) { let sameVersionOpt = Array.find( @@ -101,7 +114,6 @@ module { case (null) {}; }; - // check dependencies for (dep in config.dependencies.vals()) { let packageId = PackageUtils.getPackageId(dep.name, dep.version); if (dep.repo.size() == 0 and registry.getPackageConfig(PackageUtils.getDepName(dep.name), dep.version) == null) { @@ -112,7 +124,6 @@ module { }; }; - // check devDependencies for (dep in config.devDependencies.vals()) { let packageId = PackageUtils.getPackageId(dep.name, dep.version); if (dep.repo.size() == 0 and registry.getPackageConfig(PackageUtils.getDepName(dep.name), dep.version) == null) { @@ -120,6 +131,15 @@ module { }; }; + #ok; + }; + + public func startPublish(caller : Principal, config : PackageConfigV3) : async Result.Result { + switch (_validatePublishConfig(caller, config)) { + case (#err(err)) return #err(err); + case (#ok) {}; + }; + let publishingId = await generateId(); if (publishingPackages.get(publishingId) != null) { @@ -128,7 +148,6 @@ module { await storageManager.ensureUploadableStorages(); - // start publishingPackages.put( publishingId, { @@ -139,7 +158,33 @@ module { }, ); publishingFiles.put(publishingId, Buffer.Buffer(10)); + publishingPackageFileStats.put(publishingId, PackageUtils.defaultPackageFileStats()); + + #ok(publishingId); + }; + + public func startBlobPublish(caller : Principal, config : PackageConfigV3) : async Result.Result { + switch (_validatePublishConfig(caller, config)) { + case (#err(err)) return #err(err); + case (#ok) {}; + }; + let publishingId = await generateId(); + + if (publishingPackages.get(publishingId) != null) { + return #err("Already publishing"); + }; + + publishingPackages.put( + publishingId, + { + time = Time.now(); + user = caller; + config = config; + // Sentinel: blob packages don't use storage canisters + storage = Principal.fromText("aaaaa-aa"); + }, + ); publishingPackageFileStats.put(publishingId, PackageUtils.defaultPackageFileStats()); #ok(publishingId); @@ -498,6 +543,66 @@ module { }); }; + public func finishBlobPublish(caller : Principal, publishingId : PublishingId, blobHash : Text, archiveSize : Nat, fileCount : Nat) : async Result.Result<{ config : PackageConfigV3; publication : PackagePublication; isNewPackage : Bool }, PublishingErr> { + assert (not Principal.isAnonymous(caller)); + + if (not _isValidBlobHash(blobHash)) { + return #err("Invalid blob hash format. Expected 'sha256:<64-lowercase-hex-chars>'"); + }; + + if (archiveSize > MAX_PACKAGE_SIZE) { + return #err("Max package size is 28MB"); + }; + + if (fileCount > MAX_PACKAGE_FILES) { + return #err("Maximum number of package files: " # Nat.toText(MAX_PACKAGE_FILES)); + }; + + let ?publishing = publishingPackages.get(publishingId) else return #err("Publishing package not found"); + assert (publishing.user == caller); + + publishingPackageFileStats.put( + publishingId, + { + sourceFiles = fileCount; + sourceSize = archiveSize; + docsCount = 0; + docsSize = 0; + testFiles = 0; + testSize = 0; + benchFiles = 0; + benchSize = 0; + }, + ); + + let isNewPackage = registry.getHighestVersion(publishing.config.name) == null; + + let publication = registry.newBlobPackageRelease({ + userId = caller; + config = publishing.config; + notes = Option.get(publishingNotes.get(publishingId), ""); + blobHash = blobHash; + benchmarks = Option.get(publishingBenchmarks.get(publishingId), []); + fileStats = publishingPackageFileStats.get(publishingId); + testStats = publishingTestStats.get(publishingId); + docsCoverage = Option.get(publishingDocsCoverage.get(publishingId), 0.0); + }); + + publishingFiles.delete(publishingId); + publishingPackages.delete(publishingId); + publishingPackageFileStats.delete(publishingId); + publishingTestStats.delete(publishingId); + publishingNotes.delete(publishingId); + publishingBenchmarks.delete(publishingId); + publishingDocsCoverage.delete(publishingId); + + #ok({ + config = publishing.config; + publication; + isNewPackage; + }); + }; + func _checkPublishingPackageSize(publishingId : PublishingId) : Result.Result<(), PublishingErr> { switch (publishingPackageFileStats.get(publishingId)) { case (?fileStats) { diff --git a/backend/main/main-canister.mo b/backend/main/main-canister.mo index bf360bcc..9761593c 100644 --- a/backend/main/main-canister.mo +++ b/backend/main/main-canister.mo @@ -11,6 +11,7 @@ import Principal "mo:base/Principal"; import Order "mo:base/Order"; import Option "mo:base/Option"; import Blob "mo:base/Blob"; +import Nat8 "mo:base/Nat8"; import TelegramBot "mo:telegram-bot"; import IC "mo:ic"; @@ -60,7 +61,7 @@ actor class Main() = this { public type PublishingId = Text; public type Benchmarks = Types.Benchmarks; - let API_VERSION = "1.3"; // (!) make changes in pair with cli + let API_VERSION = "1.4"; // (!) make changes in pair with cli var packageVersions = TrieMap.TrieMap(Text.equal, Text.hash); var packageOwners = TrieMap.TrieMap(Text.equal, Text.hash); // legacy @@ -78,6 +79,13 @@ actor class Main() = this { var packageNotes = TrieMap.TrieMap(Text.equal, Text.hash); var packageDocsCoverage = TrieMap.TrieMap(Text.equal, Text.hash); + // Caffeine Object Storage state + var blobHashByPackageId = TrieMap.TrieMap(Text.equal, Text.hash); + var liveBlobHashes = TrieMap.TrieMap(Text.equal, Text.hash); + var pendingBlobDelete = TrieMap.TrieMap(Text.equal, Text.hash); + var gatewayPrincipals = TrieMap.TrieMap(Principal.equal, Principal.hash); + stable var cashierId : Principal = Principal.fromText("72ch2-fiaaa-aaaar-qbsvq-cai"); + var registry = Registry.Registry( packageVersions, ownersByPackage, @@ -92,6 +100,7 @@ actor class Main() = this { packageBenchmarks, packageNotes, packageDocsCoverage, + blobHashByPackageId, ); let downloadLog = DownloadLog.DownloadLog(); @@ -130,6 +139,14 @@ actor class Main() = this { await packagePublisher.startPublish(caller, config); }; + public shared ({ caller }) func startBlobPublish(configPub : Types.PackageConfigV3_Publishing) : async Result.Result { + let config : PackageConfigV3 = { + configPub with + requirements = Option.get(configPub.requirements, []); + }; + await packagePublisher.startBlobPublish(caller, config); + }; + public shared ({ caller }) func startFileUpload(publishingId : PublishingId, path : Text.Text, chunkCount : Nat, firstChunk : Blob) : async Result.Result { await packagePublisher.startFileUpload(caller, publishingId, path, chunkCount, firstChunk); }; @@ -169,18 +186,7 @@ actor class Main() = this { #err(err); }; case (#ok(publishResult)) { - // Send Telegram notification - let telegramBot = TelegramBot.TelegramBot(telegramBotToken, transformTelegramRequest); - let message = _formatTelegramMessage(publishResult.config, publishResult.publication, publishResult.isNewPackage); - let tgRes = await telegramBot.sendMessage("@mops_feed", message, null); - - switch (tgRes) { - case (#err(err)) { - Debug.print("Failed to send message to telegram: " # err); - }; - case (#ok) {}; - }; - + await _sendTelegramNotification(publishResult.config, publishResult.publication, publishResult.isNewPackage); #ok; }; }; @@ -219,6 +225,18 @@ actor class Main() = this { }; }; + func _sendTelegramNotification(config : PackageConfigV3, publication : PackagePublication, isNewPackage : Bool) : async () { + let telegramBot = TelegramBot.TelegramBot(telegramBotToken, transformTelegramRequest); + let message = _formatTelegramMessage(config, publication, isNewPackage); + let tgRes = await telegramBot.sendMessage("@mops_feed", message, null); + switch (tgRes) { + case (#err(err)) { + Debug.print("Failed to send message to telegram: " # err); + }; + case (#ok) {}; + }; + }; + public shared ({ caller }) func computeHashesForExistingFiles() : async () { assert (Utils.isAdmin(caller)); @@ -242,6 +260,134 @@ actor class Main() = this { }; }; + // CAFFEINE OBJECT STORAGE PROTOCOL + + public type CreateCertificateResult = Types.CreateCertificateResult; + + func _rebuildLiveBlobHashes() { + liveBlobHashes := TrieMap.TrieMap(Text.equal, Text.hash); + for ((_, blobHash) in blobHashByPackageId.entries()) { + liveBlobHashes.put(blobHash, ()); + }; + }; + + func _bytesToHash(bytes : Blob) : ?Text { + let arr = Blob.toArray(bytes); + if (arr.size() != 32) return null; + let hexDigits : [Char] = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']; + var hex = "sha256:"; + for (b in arr.vals()) { + let n = Nat8.toNat(b); + hex #= Text.fromChar(hexDigits[n / 16]); + hex #= Text.fromChar(hexDigits[n % 16]); + }; + ?hex; + }; + + func _isValidBlobHash(hash : Text) : Bool { + if (Text.size(hash) != 71) return false; + if (not Text.startsWith(hash, #text("sha256:"))) return false; + let hexPart = switch (Text.stripStart(hash, #text("sha256:"))) { + case null return false; + case (?h) h; + }; + for (c in hexPart.chars()) { + if (not ((c >= '0' and c <= '9') or (c >= 'a' and c <= 'f'))) { + return false; + }; + }; + true; + }; + + func _callerIsGateway(caller : Principal) : Bool { + if (Principal.isAnonymous(caller)) return false; + gatewayPrincipals.get(caller) != null; + }; + + public shared func _immutableObjectStorageCreateCertificate(hash : Text) : async CreateCertificateResult { + if (not _isValidBlobHash(hash)) { + Debug.trap("hash must be 'sha256:<64-lowercase-hex-chars>'"); + }; + + if (liveBlobHashes.get(hash) != null) { + pendingBlobDelete.delete(hash); + }; + + { method = "upload"; blob_hash = hash }; + }; + + public shared query func _immutableObjectStorageBlobsAreLive(hashBytesList : [Blob]) : async [Bool] { + Array.map( + hashBytesList, + func(hashBytes : Blob) : Bool { + switch (_bytesToHash(hashBytes)) { + case null false; + case (?hash) { + liveBlobHashes.get(hash) != null and pendingBlobDelete.get(hash) == null; + }; + }; + }, + ); + }; + + public shared query ({ caller }) func _immutableObjectStorageBlobsToDelete() : async [Text] { + if (not _callerIsGateway(caller)) return []; + Iter.toArray(pendingBlobDelete.keys()); + }; + + public shared ({ caller }) func _immutableObjectStorageConfirmBlobDeletion(hashBytesList : [Blob]) : async () { + if (not _callerIsGateway(caller)) return; + for (hashBytes in hashBytesList.vals()) { + switch (_bytesToHash(hashBytes)) { + case null {}; + case (?hash) { + pendingBlobDelete.delete(hash); + }; + }; + }; + }; + + public shared func _immutableObjectStorageUpdateGatewayPrincipals() : async () { + let cashier : actor { + storage_gateway_principal_list_v1 : shared query () -> async [Principal]; + } = actor (Principal.toText(cashierId)); + let principals = await cashier.storage_gateway_principal_list_v1(); + let existing = Iter.toArray(gatewayPrincipals.keys()); + for (p in existing.vals()) { + gatewayPrincipals.delete(p); + }; + for (p in principals.vals()) { + gatewayPrincipals.put(p, ()); + }; + }; + + // Blob publish + public shared ({ caller }) func finishBlobPublish(publishingId : PublishingId, blobHash : Text, archiveSize : Nat, fileCount : Nat) : async Result.Result<(), Err> { + let res = await packagePublisher.finishBlobPublish(caller, publishingId, blobHash, archiveSize, fileCount); + + switch (res) { + case (#err(err)) { + #err(err); + }; + case (#ok(publishResult)) { + liveBlobHashes.put(blobHash, ()); + + await _sendTelegramNotification(publishResult.config, publishResult.publication, publishResult.isNewPackage); + #ok; + }; + }; + }; + + public query func getBlobHash(name : PackageName, version : PackageVersion) : async ?Text { + let packageId = PackageUtils.getPackageId(name, version); + blobHashByPackageId.get(packageId); + }; + + public shared ({ caller }) func setCashierId(newCashierId : Principal) : async () { + assert (Utils.isAdmin(caller)); + cashierId := newCashierId; + }; + public shared ({ caller }) func setStorageControllers() : async () { assert (Utils.isAdmin(caller)); let self = Principal.fromActor(this); @@ -346,7 +492,16 @@ actor class Main() = this { public query func getFileIds(name : PackageName, version : PackageVersion) : async Result.Result<[FileId], Err> { let packageId = PackageUtils.getPackageId(name, version); - Result.fromOption(fileIdsByPackage.get(packageId), "Package '" # packageId # "' not found"); + switch (fileIdsByPackage.get(packageId)) { + case (?ids) #ok(ids); + case null { + if (blobHashByPackageId.get(packageId) != null) { + #err("Package '" # packageId # "' uses blob storage. Please upgrade mops CLI to the latest version."); + } else { + #err("Package '" # packageId # "' not found"); + }; + }; + }; }; func _getFileHashes(packageId : PackageId) : Result.Result<[(FileId, Blob)], Err> { @@ -663,6 +818,11 @@ actor class Main() = this { #storageManager : StorageManager.Stable; #users : Users.Stable; }; + #v10 : { + #blobHashByPackageId : [(PackageId, Text)]; + #pendingBlobDelete : [(Text, ())]; + #gatewayPrincipals : [(Principal, ())]; + }; }; public shared ({ caller }) func backup() : async () { @@ -676,7 +836,7 @@ actor class Main() = this { }; func _backup() : async () { - let backup = backupManager.NewBackup("v9"); + let backup = backupManager.NewBackup("v10"); await backup.startBackup(); await backup.uploadChunk(to_candid (#v9(#packagePublications(Iter.toArray(packagePublications.entries()))) : BackupChunk)); await backup.uploadChunk(to_candid (#v9(#packageVersions(Iter.toArray(packageVersions.entries()))) : BackupChunk)); @@ -694,6 +854,9 @@ actor class Main() = this { await backup.uploadChunk(to_candid (#v9(#highestConfigs(Iter.toArray(highestConfigs.entries()))) : BackupChunk)); await backup.uploadChunk(to_candid (#v9(#packageConfigs(Iter.toArray(packageConfigs.entries()))) : BackupChunk)); await backup.uploadChunk(to_candid (#v9(#packageDocsCoverage(Iter.toArray(packageDocsCoverage.entries()))) : BackupChunk)); + await backup.uploadChunk(to_candid (#v10(#blobHashByPackageId(Iter.toArray(blobHashByPackageId.entries()))) : BackupChunk)); + await backup.uploadChunk(to_candid (#v10(#pendingBlobDelete(Iter.toArray(pendingBlobDelete.entries()))) : BackupChunk)); + await backup.uploadChunk(to_candid (#v10(#gatewayPrincipals(Iter.toArray(gatewayPrincipals.entries()))) : BackupChunk)); await backup.finishBackup(); }; @@ -705,57 +868,74 @@ actor class Main() = this { await backupManager.restore( backupId, func(blob : Blob) { - let ?#v9(chunk) : ?BackupChunk = from_candid (blob) else Debug.trap("Failed to restore chunk"); - - switch (chunk) { - case (#packagePublications(packagePublicationsStable)) { - packagePublications := TrieMap.fromEntries(packagePublicationsStable.vals(), Text.equal, Text.hash); - }; - case (#packageVersions(packageVersionsStable)) { - packageVersions := TrieMap.fromEntries(packageVersionsStable.vals(), Text.equal, Text.hash); - }; - case (#ownersByPackage(ownersByPackageStable)) { - ownersByPackage := TrieMap.fromEntries(ownersByPackageStable.vals(), Text.equal, Text.hash); - }; - case (#maintainersByPackage(maintainersByPackageStable)) { - maintainersByPackage := TrieMap.fromEntries(maintainersByPackageStable.vals(), Text.equal, Text.hash); - }; - case (#fileIdsByPackage(fileIdsByPackageStable)) { - fileIdsByPackage := TrieMap.fromEntries(fileIdsByPackageStable.vals(), Text.equal, Text.hash); - }; - case (#hashByFileId(hashByFileIdStable)) { - hashByFileId := TrieMap.fromEntries(hashByFileIdStable.vals(), Text.equal, Text.hash); - }; - case (#packageFileStats(packageFileStatsStable)) { - packageFileStats := TrieMap.fromEntries(packageFileStatsStable.vals(), Text.equal, Text.hash); - }; - case (#packageTestStats(packageTestStatsStable)) { - packageTestStats := TrieMap.fromEntries(packageTestStatsStable.vals(), Text.equal, Text.hash); - }; - case (#packageBenchmarks(packageBenchmarksStable)) { - packageBenchmarks := TrieMap.fromEntries(packageBenchmarksStable.vals(), Text.equal, Text.hash); - }; - case (#packageNotes(packageNotesStable)) { - packageNotes := TrieMap.fromEntries(packageNotesStable.vals(), Text.equal, Text.hash); + let ?backupChunk : ?BackupChunk = from_candid (blob) else Debug.trap("Failed to restore chunk"); + + switch (backupChunk) { + case (#v9(chunk)) { + switch (chunk) { + case (#packagePublications(data)) { + packagePublications := TrieMap.fromEntries(data.vals(), Text.equal, Text.hash); + }; + case (#packageVersions(data)) { + packageVersions := TrieMap.fromEntries(data.vals(), Text.equal, Text.hash); + }; + case (#ownersByPackage(data)) { + ownersByPackage := TrieMap.fromEntries(data.vals(), Text.equal, Text.hash); + }; + case (#maintainersByPackage(data)) { + maintainersByPackage := TrieMap.fromEntries(data.vals(), Text.equal, Text.hash); + }; + case (#fileIdsByPackage(data)) { + fileIdsByPackage := TrieMap.fromEntries(data.vals(), Text.equal, Text.hash); + }; + case (#hashByFileId(data)) { + hashByFileId := TrieMap.fromEntries(data.vals(), Text.equal, Text.hash); + }; + case (#packageFileStats(data)) { + packageFileStats := TrieMap.fromEntries(data.vals(), Text.equal, Text.hash); + }; + case (#packageTestStats(data)) { + packageTestStats := TrieMap.fromEntries(data.vals(), Text.equal, Text.hash); + }; + case (#packageBenchmarks(data)) { + packageBenchmarks := TrieMap.fromEntries(data.vals(), Text.equal, Text.hash); + }; + case (#packageNotes(data)) { + packageNotes := TrieMap.fromEntries(data.vals(), Text.equal, Text.hash); + }; + case (#packageDocsCoverage(data)) { + packageDocsCoverage := TrieMap.fromEntries(data.vals(), Text.equal, Text.hash); + }; + case (#downloadLog(data)) { + downloadLog.cancelTimers(); + downloadLog.loadStable(data); + }; + case (#storageManager(data)) { + storageManager.loadStable(data); + }; + case (#users(data)) { + users.loadStable(data); + }; + case (#highestConfigs(data)) { + highestConfigs := TrieMap.fromEntries(data.vals(), Text.equal, Text.hash); + }; + case (#packageConfigs(data)) { + packageConfigs := TrieMap.fromEntries(data.vals(), Text.equal, Text.hash); + }; + }; }; - case (#packageDocsCoverage(packageDocsCoverageStable)) { - packageDocsCoverage := TrieMap.fromEntries(packageDocsCoverageStable.vals(), Text.equal, Text.hash); - }; - case (#downloadLog(downloadLogStable)) { - downloadLog.cancelTimers(); - downloadLog.loadStable(downloadLogStable); - }; - case (#storageManager(storageManagerStable)) { - storageManager.loadStable(storageManagerStable); - }; - case (#users(usersStable)) { - users.loadStable(usersStable); - }; - case (#highestConfigs(highestConfigsStable)) { - highestConfigs := TrieMap.fromEntries(highestConfigsStable.vals(), Text.equal, Text.hash); - }; - case (#packageConfigs(packageConfigsStable)) { - packageConfigs := TrieMap.fromEntries(packageConfigsStable.vals(), Text.equal, Text.hash); + case (#v10(chunk)) { + switch (chunk) { + case (#blobHashByPackageId(data)) { + blobHashByPackageId := TrieMap.fromEntries(data.vals(), Text.equal, Text.hash); + }; + case (#pendingBlobDelete(data)) { + pendingBlobDelete := TrieMap.fromEntries(data.vals(), Text.equal, Text.hash); + }; + case (#gatewayPrincipals(data)) { + gatewayPrincipals := TrieMap.fromEntries(data.vals(), Principal.equal, Principal.hash); + }; + }; }; }; }, @@ -763,6 +943,8 @@ actor class Main() = this { downloadLog.setTimers(); + _rebuildLiveBlobHashes(); + // re-init registry registry := Registry.Registry( packageVersions, @@ -778,6 +960,7 @@ actor class Main() = this { packageBenchmarks, packageNotes, packageDocsCoverage, + blobHashByPackageId, ); packagePublisher := PackagePublisher.PackagePublisher(registry, storageManager); }; @@ -804,6 +987,10 @@ actor class Main() = this { stable var storageManagerStable : StorageManager.Stable = null; stable var usersStable : Users.Stable = null; + stable var blobHashByPackageIdStable : [(PackageId, Text)] = []; + stable var pendingBlobDeleteStable : [(Text, ())] = []; + stable var gatewayPrincipalsStable : [(Principal, ())] = []; + system func preupgrade() { packagePublicationsStable := Iter.toArray(packagePublications.entries()); packageVersionsStable := Iter.toArray(packageVersions.entries()); @@ -824,6 +1011,10 @@ actor class Main() = this { highestConfigsStableV3 := Iter.toArray(highestConfigs.entries()); packageConfigsStableV3 := Iter.toArray(packageConfigs.entries()); + + blobHashByPackageIdStable := Iter.toArray(blobHashByPackageId.entries()); + pendingBlobDeleteStable := Iter.toArray(pendingBlobDelete.entries()); + gatewayPrincipalsStable := Iter.toArray(gatewayPrincipals.entries()); }; system func postupgrade() { @@ -887,6 +1078,17 @@ actor class Main() = this { users.loadStable(usersStable); usersStable := null; + blobHashByPackageId := TrieMap.fromEntries(blobHashByPackageIdStable.vals(), Text.equal, Text.hash); + blobHashByPackageIdStable := []; + + pendingBlobDelete := TrieMap.fromEntries(pendingBlobDeleteStable.vals(), Text.equal, Text.hash); + pendingBlobDeleteStable := []; + + gatewayPrincipals := TrieMap.fromEntries(gatewayPrincipalsStable.vals(), Principal.equal, Principal.hash); + gatewayPrincipalsStable := []; + + _rebuildLiveBlobHashes(); + registry := Registry.Registry( packageVersions, ownersByPackage, @@ -901,6 +1103,7 @@ actor class Main() = this { packageBenchmarks, packageNotes, packageDocsCoverage, + blobHashByPackageId, ); packagePublisher := PackagePublisher.PackagePublisher(registry, storageManager); diff --git a/backend/main/registry/Registry.mo b/backend/main/registry/Registry.mo index 2aba02ad..7edb6226 100644 --- a/backend/main/registry/Registry.mo +++ b/backend/main/registry/Registry.mo @@ -34,6 +34,17 @@ module { docsCoverage : Float; }; + public type NewBlobPackageReleaseArgs = { + userId : Principal; + config : PackageConfigV3; + notes : Text; + blobHash : Text; + fileStats : ?PackageFileStats; + testStats : ?TestStats; + benchmarks : Benchmarks; + docsCoverage : Float; + }; + public class Registry( packageVersions : TrieMap.TrieMap, ownersByPackage : TrieMap.TrieMap, @@ -48,6 +59,7 @@ module { packageBenchmarks : TrieMap.TrieMap, packageNotes : TrieMap.TrieMap, packageDocsCoverage : TrieMap.TrieMap, + blobHashByPackageId : TrieMap.TrieMap, ) { // ----------------------------- @@ -104,6 +116,54 @@ module { publication; }; + public func newBlobPackageRelease(newRelease : NewBlobPackageReleaseArgs) : PackagePublication { + let packageId = PackageUtils.getPackageId(newRelease.config.name, newRelease.config.version); + + _updateHighestConfig(newRelease.config); + + let versions = Option.get(packageVersions.get(newRelease.config.name), []); + packageVersions.put(newRelease.config.name, Array.append(versions, [newRelease.config.version])); + + packageConfigs.put(packageId, newRelease.config); + + let owners = getPackageOwners(newRelease.config.name); + if (owners.size() == 0) { + ownersByPackage.put(newRelease.config.name, [newRelease.userId]); + }; + + let publication = { + user = newRelease.userId; + time = Time.now(); + // Blob packages don't use storage canisters; use management canister + // principal as a sentinel (PackagePublication requires this field). + // Clients must check getBlobHash() before accessing this value. + storage = Principal.fromText("aaaaa-aa"); + }; + packagePublications.put(packageId, publication); + + blobHashByPackageId.put(packageId, newRelease.blobHash); + + switch (newRelease.fileStats) { + case (?fileStats) { + packageFileStats.put(packageId, fileStats); + }; + case (null) {}; + }; + + switch (newRelease.testStats) { + case (?testStats) { + packageTestStats.put(packageId, testStats); + }; + case (null) {}; + }; + + packageBenchmarks.put(packageId, newRelease.benchmarks); + packageNotes.put(packageId, newRelease.notes); + packageDocsCoverage.put(packageId, newRelease.docsCoverage); + + publication; + }; + func _updateHighestConfig(config : PackageConfigV3) { switch (getHighestVersion(config.name)) { case (?ver) { diff --git a/backend/main/types.mo b/backend/main/types.mo index 6a3a779f..a78c5c83 100644 --- a/backend/main/types.mo +++ b/backend/main/types.mo @@ -192,6 +192,11 @@ module { #tooOld; }; + public type CreateCertificateResult = { + method : Text; + blob_hash : Text; + }; + public type Benchmarks = [Benchmark]; type BenchmarkMetric = Text; // instructions, rts_heap_size, rts_logical_stable_memory_size, rts_reclaimed diff --git a/cli/api/downloadPackageFiles.ts b/cli/api/downloadPackageFiles.ts index b585ccd1..f1d17aba 100644 --- a/cli/api/downloadPackageFiles.ts +++ b/cli/api/downloadPackageFiles.ts @@ -1,8 +1,11 @@ +import path from "node:path"; import { Principal } from "@icp-sdk/core/principal"; +import { Parser as TarParser, type ReadEntry } from "tar"; import { mainActor, storageActor } from "./actors.js"; import { resolveVersion } from "./resolveVersion.js"; import { parallel } from "../parallel.js"; import { Storage } from "../declarations/storage/storage.did.js"; +import { downloadBlob, verifyBlobHash } from "./storageClient.js"; export async function downloadPackageFiles( pkg: string, @@ -12,6 +15,71 @@ export async function downloadPackageFiles( ): Promise>> { version = await resolveVersion(pkg, version); + let actor = await mainActor(); + let blobHash = await actor.getBlobHash(pkg, version); + + if (blobHash.length > 0 && blobHash[0]) { + return await downloadBlobPackage(blobHash[0]); + } + + return await downloadLegacyPackage(pkg, version, threads, onLoad); +} + +async function downloadBlobPackage( + blobHash: string, +): Promise>> { + let archiveData = await downloadBlob(blobHash); + + verifyBlobHash(archiveData, blobHash); + + let filesData = new Map>(); + + await new Promise((resolve, reject) => { + let parser = new TarParser(); + parser.on("entry", (entry: ReadEntry) => { + if (entry.type !== "File") { + entry.resume(); + return; + } + let entryPath = sanitizeTarPath(entry.path); + if (!entryPath) { + entry.resume(); + return; + } + let chunks: Buffer[] = []; + entry.on("data", (chunk: Buffer) => chunks.push(chunk)); + entry.on("end", () => { + let data = Buffer.concat(chunks); + filesData.set(entryPath, Array.from(data)); + }); + }); + parser.on("end", resolve); + parser.on("error", reject); + parser.write(Buffer.from(archiveData)); + parser.end(); + }); + + return filesData; +} + +function sanitizeTarPath(entryPath: string): string | null { + let normalized = path.normalize(entryPath); + if ( + path.isAbsolute(normalized) || + normalized.startsWith("..") || + normalized.includes(`..${path.sep}`) + ) { + return null; + } + return normalized; +} + +async function downloadLegacyPackage( + pkg: string, + version: string, + threads: number, + onLoad: (_fileIds: string[], _fileId: string) => void, +): Promise>> { let { storageId, fileIds } = await getPackageFilesInfo(pkg, version); let storage = await storageActor(storageId); diff --git a/cli/api/storageClient.ts b/cli/api/storageClient.ts new file mode 100644 index 00000000..c340fc97 --- /dev/null +++ b/cli/api/storageClient.ts @@ -0,0 +1,385 @@ +import { sha256 } from "@noble/hashes/sha256"; +import { bytesToHex } from "@noble/hashes/utils"; +import { + HttpAgent, + type Identity, + isV3ResponseBody, +} from "@icp-sdk/core/agent"; +import { IDL } from "@icp-sdk/core/candid"; +import { getEndpoint, getNetwork } from "./network.js"; + +const SHA256_PREFIX = "sha256:"; +const CHUNK_SIZE = 1024 * 1024; // 1 MiB +const MAX_CONCURRENT_UPLOADS = 10; +const MAX_RETRIES = 3; +const BASE_DELAY_MS = 1000; +const FETCH_TIMEOUT_MS = 120_000; +const GATEWAY_VERSION = "v1"; + +const DOMAIN_CHUNK = new TextEncoder().encode("icfs-chunk/"); +const DOMAIN_METADATA = new TextEncoder().encode("icfs-metadata/"); +const DOMAIN_NODE = new TextEncoder().encode("ynode/"); +const UNBALANCED = new TextEncoder().encode("UNBALANCED"); + +function domainHash(domainSeparator: Uint8Array, data: Uint8Array): Uint8Array { + const combined = new Uint8Array(domainSeparator.length + data.length); + combined.set(domainSeparator); + combined.set(data, domainSeparator.length); + return sha256(combined); +} + +function chunkHash(data: Uint8Array): Uint8Array { + return domainHash(DOMAIN_CHUNK, data); +} + +function nodeHash( + left: Uint8Array | null, + right: Uint8Array | null, +): Uint8Array { + const leftBytes = left ?? UNBALANCED; + const rightBytes = right ?? UNBALANCED; + const combined = new Uint8Array( + DOMAIN_NODE.length + leftBytes.length + rightBytes.length, + ); + let offset = 0; + for (const data of [DOMAIN_NODE, leftBytes, rightBytes]) { + combined.set(data, offset); + offset += data.length; + } + return sha256(combined); +} + +function metadataHash(headers: Record): Uint8Array { + const lines: string[] = []; + for (const [key, value] of Object.entries(headers)) { + lines.push(`${key.trim()}: ${value.trim()}\n`); + } + lines.sort(); + return domainHash(DOMAIN_METADATA, new TextEncoder().encode(lines.join(""))); +} + +function hashToShaString(hash: Uint8Array): string { + return `${SHA256_PREFIX}${bytesToHex(hash)}`; +} + +type TreeNode = { + hash: Uint8Array; + left: TreeNode | null; + right: TreeNode | null; +}; + +type TreeNodeJSON = { + hash: string; + left: TreeNodeJSON | null; + right: TreeNodeJSON | null; +}; + +function nodeToJSON(node: TreeNode): TreeNodeJSON { + return { + hash: hashToShaString(node.hash), + left: node.left ? nodeToJSON(node.left) : null, + right: node.right ? nodeToJSON(node.right) : null, + }; +} + +type BlobHashTreeJSON = { + tree_type: "DSBMTWH"; + chunk_hashes: string[]; + tree: TreeNodeJSON; + headers: string[]; +}; + +function splitChunks(data: Uint8Array): Uint8Array[] { + const chunks: Uint8Array[] = []; + for (let i = 0; i < data.length; i += CHUNK_SIZE) { + chunks.push(data.subarray(i, Math.min(i + CHUNK_SIZE, data.length))); + } + return chunks; +} + +function buildMerkleTree( + fileData: Uint8Array, + contentType: string, +): { + chunks: Uint8Array[]; + chunkHashes: Uint8Array[]; + blobTree: BlobHashTreeJSON; + rootHash: string; +} { + if (fileData.length === 0) { + throw new Error("Cannot build merkle tree from empty data"); + } + + const chunks = splitChunks(fileData); + const chunkHashes = chunks.map((c) => chunkHash(c)); + + let level: TreeNode[] = chunkHashes.map((h) => ({ + hash: h, + left: null, + right: null, + })); + + while (level.length > 1) { + const nextLevel: TreeNode[] = []; + for (let i = 0; i < level.length; i += 2) { + const left = level[i]!; + const right = level[i + 1] ?? null; + const parent = nodeHash(left.hash, right ? right.hash : null); + nextLevel.push({ hash: parent, left, right }); + } + level = nextLevel; + } + + const chunksRoot = level[0]!; + + const headers: Record = { + "Content-Type": contentType, + "Content-Length": fileData.length.toString(), + }; + const headerLines = Object.entries(headers).map( + ([k, v]) => `${k.trim()}: ${v.trim()}`, + ); + headerLines.sort(); + + const metaHash = metadataHash(headers); + const metaNode: TreeNode = { hash: metaHash, left: null, right: null }; + const combinedHash = nodeHash(chunksRoot.hash, metaNode.hash); + const combinedRoot: TreeNode = { + hash: combinedHash, + left: chunksRoot, + right: metaNode, + }; + + const blobTree: BlobHashTreeJSON = { + tree_type: "DSBMTWH", + chunk_hashes: chunkHashes.map(hashToShaString), + tree: nodeToJSON(combinedRoot), + headers: headerLines, + }; + + return { + chunks, + chunkHashes, + blobTree, + rootHash: hashToShaString(combinedRoot.hash), + }; +} + +async function withRetry(operation: () => Promise): Promise { + let lastError: Error | undefined; + for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { + try { + return await operation(); + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)); + if (attempt === MAX_RETRIES) { + throw error; + } + const delay = Math.min( + BASE_DELAY_MS * Math.pow(2, attempt) + Math.random() * 1000, + 30000, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } + throw lastError || new Error("Unknown error during retry"); +} + +export async function getCertificate( + identity: Identity, + rootHash: string, +): Promise { + const network = getNetwork(); + const { host, canisterId } = getEndpoint(network); + + const agent = await HttpAgent.create({ + host, + identity, + shouldFetchRootKey: network === "local", + verifyQuerySignatures: process.env.MOPS_VERIFY_QUERY_SIGNATURES !== "false", + shouldSyncTime: true, + }); + + const arg = IDL.encode([IDL.Text], [rootHash]); + const result = await agent.call(canisterId, { + methodName: "_immutableObjectStorageCreateCertificate", + arg, + }); + const body = result.response.body; + if (isV3ResponseBody(body)) { + return body.certificate; + } + throw new Error("Expected v3 response body with certificate"); +} + +function fetchWithTimeout( + url: string | URL, + init?: RequestInit, +): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); + return fetch(url, { ...init, signal: controller.signal }).finally(() => + clearTimeout(timeout), + ); +} + +function getGatewayUrl(): string { + return process.env["MOPS_STORAGE_GATEWAY_URL"] || "https://blob.caffeine.ai"; +} + +function getProjectId(): string { + return ( + process.env["MOPS_STORAGE_PROJECT_ID"] || + "00000000-0000-0000-0000-000000000000" + ); +} + +export async function uploadBlob( + fileData: Uint8Array, + identity: Identity, + onProgress?: (percentage: number) => void, +): Promise { + const { chunks, chunkHashes, blobTree, rootHash } = buildMerkleTree( + fileData, + "application/gzip", + ); + + const certificateBytes = await getCertificate(identity, rootHash); + + const gatewayUrl = getGatewayUrl(); + const network = getNetwork(); + const { canisterId } = getEndpoint(network); + const projectId = getProjectId(); + + await withRetry(async () => { + const url = `${gatewayUrl}/${GATEWAY_VERSION}/blob-tree/`; + const requestBody = { + blob_tree: blobTree, + bucket_name: "default-bucket", + num_blob_bytes: fileData.length, + owner: canisterId, + project_id: projectId, + headers: blobTree.headers, + auth: { + OwnerEgressSignature: Array.from(certificateBytes), + }, + }; + + const response = await fetchWithTimeout(url, { + method: "PUT", + headers: { + "Content-Type": "application/json", + "X-Caffeine-Project-ID": projectId, + }, + body: JSON.stringify(requestBody), + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error( + `Failed to upload blob tree: ${response.status} ${response.statusText} - ${errorText}`, + ); + } + }); + + let completedChunks = 0; + + const uploadChunk = async (index: number) => { + const chunkData = chunks[index]!; + const blobRootHash = rootHash; + const chunkHashStr = hashToShaString(chunkHashes[index]!); + + await withRetry(async () => { + const queryParams = new URLSearchParams({ + owner_id: canisterId, + blob_hash: blobRootHash, + chunk_hash: chunkHashStr, + chunk_index: index.toString(), + bucket_name: "default-bucket", + project_id: projectId, + }); + const url = `${gatewayUrl}/${GATEWAY_VERSION}/chunk/?${queryParams.toString()}`; + + const response = await fetchWithTimeout(url, { + method: "PUT", + headers: { + "Content-Type": "application/octet-stream", + "X-Caffeine-Project-ID": projectId, + }, + body: chunkData, + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error( + `Failed to upload chunk ${index}: ${response.status} ${response.statusText} - ${errorText}`, + ); + } + }); + + completedChunks++; + if (onProgress) { + onProgress( + chunks.length === 0 + ? 100 + : Math.round((completedChunks / chunks.length) * 100), + ); + } + }; + + await Promise.all( + Array.from({ length: MAX_CONCURRENT_UPLOADS }, async (_, workerId) => { + for (let i = workerId; i < chunks.length; i += MAX_CONCURRENT_UPLOADS) { + await uploadChunk(i); + } + }), + ); + + return rootHash; +} + +export function getDownloadUrl(blobHash: string): string { + const gatewayUrl = getGatewayUrl(); + const network = getNetwork(); + const { canisterId } = getEndpoint(network); + const projectId = getProjectId(); + return ( + `${gatewayUrl}/${GATEWAY_VERSION}/blob/` + + `?blob_hash=${encodeURIComponent(blobHash)}` + + `&owner_id=${encodeURIComponent(canisterId)}` + + `&project_id=${encodeURIComponent(projectId)}` + ); +} + +export async function downloadBlob(blobHash: string): Promise { + const url = getDownloadUrl(blobHash); + const response = await withRetry(async () => { + const res = await fetchWithTimeout(url); + if (!res.ok) { + throw new Error( + `Failed to download blob: ${res.status} ${res.statusText}`, + ); + } + return res; + }); + const buffer = await response.arrayBuffer(); + return new Uint8Array(buffer); +} + +export function verifyBlobHash(data: Uint8Array, expectedHash: string): void { + const { rootHash } = buildMerkleTree(data, "application/gzip"); + if (rootHash !== expectedHash) { + throw new Error( + `Blob integrity check failed: expected ${expectedHash} but got ${rootHash}`, + ); + } +} + +export { + buildMerkleTree, + splitChunks, + chunkHash, + nodeHash, + metadataHash, + hashToShaString, +}; diff --git a/cli/commands/install/install-mops-dep.ts b/cli/commands/install/install-mops-dep.ts index 53b71431..794c90bf 100644 --- a/cli/commands/install/install-mops-dep.ts +++ b/cli/commands/install/install-mops-dep.ts @@ -110,14 +110,17 @@ export async function installMopsDep( try { await Promise.all( Array.from(filesData.entries()).map(async ([filePath, data]) => { - await fs.promises.mkdir( - path.join(cacheDir, path.dirname(filePath)), - { recursive: true }, - ); - await fs.promises.writeFile( - path.join(cacheDir, filePath), - Buffer.from(data), - ); + let resolvedPath = path.resolve(cacheDir, filePath); + if ( + !resolvedPath.startsWith(cacheDir + path.sep) && + resolvedPath !== cacheDir + ) { + throw new Error(`Path traversal detected: ${filePath}`); + } + await fs.promises.mkdir(path.dirname(resolvedPath), { + recursive: true, + }); + await fs.promises.writeFile(resolvedPath, Buffer.from(data)); }), ); } catch (err) { diff --git a/cli/commands/publish.ts b/cli/commands/publish.ts index 79814c71..411636fb 100644 --- a/cli/commands/publish.ts +++ b/cli/commands/publish.ts @@ -6,6 +6,7 @@ import logUpdate from "log-update"; import { globbySync } from "globby"; import { minimatch } from "minimatch"; import prompts from "prompts"; +import { create as tarCreate } from "tar"; import { checkConfigFile, @@ -15,7 +16,6 @@ import { readConfig, } from "../mops.js"; import { mainActor } from "../api/actors.js"; -import { parallel } from "../parallel.js"; import { docs } from "./docs.js"; import { Benchmarks, @@ -29,6 +29,7 @@ import { SilentReporter } from "./test/reporters/silent-reporter.js"; import { findChangelogEntry } from "../helpers/find-changelog-entry.js"; import { bench } from "./bench.js"; import { docsCoverage } from "./docs-coverage.js"; +import { uploadBlob } from "../api/storageClient.js"; export async function publish( options: { @@ -287,7 +288,7 @@ export async function publish( ]; let files = config.package.files || ["**/*.mo"]; files = [...files, ...defaultFiles]; - files = globbySync([...files, ...defaultFiles]); + files = globbySync([...files, ...defaultFiles], { cwd: rootDir }); if (options.verbose) { console.log("Files:"); @@ -396,27 +397,49 @@ export async function publish( } } - // progress - let total = files.length + 2; + let identity = await getIdentity(); + if (!identity) { + console.log( + chalk.red("Error: ") + + "Identity not found. Please run `mops init` first.", + ); + process.exit(1); + } + let actor = await mainActor(identity); + + // Create tar.gz archive of package files (excluding docs.tgz) + let sourceFiles = files.filter((f) => f !== docsFile); + console.log("Creating package archive..."); + let archivePath = path.join(rootDir, ".mops/.publish-archive.tar.gz"); + fs.mkdirSync(path.dirname(archivePath), { recursive: true }); + await tarCreate( + { + gzip: true, + file: archivePath, + cwd: rootDir, + portable: true, + }, + sourceFiles, + ); + let archiveData = new Uint8Array(fs.readFileSync(archivePath)); + fs.rmSync(archivePath, { force: true }); + + let total = 4; let step = 0; - function progress() { + function progress(label = "Publishing") { step++; - logUpdate(`Uploading files ${progressBar(step, total)}`); + logUpdate(`${label} ${progressBar(step, total)}`); } - // upload config - let identity = await getIdentity(); - let actor = await mainActor(identity); - - progress(); - let publishing = await actor.startPublish(backendPkgConfig); + // start blob publish + progress("Starting publish"); + let publishing = await actor.startBlobPublish(backendPkgConfig); if ("err" in publishing) { console.log(chalk.red("Error: ") + publishing.err); process.exit(1); } let publishingId = publishing.ok; - // upload test stats if (options.test) { await actor.uploadTestStats(publishingId, { passed: BigInt(reporter.passed), @@ -424,78 +447,59 @@ export async function publish( }); } - // upload benchmarks if (options.bench) { await actor.uploadBenchmarks(publishingId, benchmarks); } - // upload changelog if (changelog) { await actor.uploadNotes(publishingId, changelog); } - // upload docs coverage if (options.docs) { await actor.uploadDocsCoverage(publishingId, docsCov); } - // upload files - await parallel(8, files, async (file: string) => { - progress(); - - let chunkSize = 1024 * 1024 + 512 * 1024; // 1.5mb - let content = fs.readFileSync(file); - let chunkCount = Math.ceil(content.length / chunkSize); - let firstChunk = Array.from(content.slice(0, chunkSize)); - - // remove path from docs file - if (file === docsFile) { - file = path.basename(file); - } - - let res = await actor.startFileUpload( - publishingId, - file, - BigInt(chunkCount), - firstChunk, - ); - if ("err" in res) { - console.log(chalk.red("Error: ") + res.err); - process.exit(1); - } - let fileId = res.ok; - - for (let i = 1; i < chunkCount; i++) { - let start = i * chunkSize; - let chunk = Array.from(content.slice(start, start + chunkSize)); - let res = await actor.uploadFileChunk( - publishingId, - fileId, - BigInt(i), - chunk, + // upload to Caffeine Object Storage + progress("Uploading to blob storage"); + let rootHash: string; + const uploadStepStart = (step / total) * 100; + const uploadStepSize = (1 / total) * 100; + try { + rootHash = await uploadBlob(archiveData, identity, (pct) => { + const uploadProgress = Math.min( + 100, + Math.round(uploadStepStart + (pct / 100) * uploadStepSize), ); - if ("err" in res) { - console.log(chalk.red("Error: ") + res.err); - process.exit(1); - } - } - }); + logUpdate( + `Uploading to blob storage ${progressBar(uploadProgress, 100)}`, + ); + }); + } catch (err) { + console.log(chalk.red("Error: ") + `Failed to upload blob: ${err}`); + process.exit(1); + } + + progress("Finishing publish"); fs.rmSync(path.join(rootDir, ".mops/.docs"), { force: true, recursive: true, }); - // finish - progress(); - logUpdate.done(); - - let res = await actor.finishPublish(publishingId); + let res = await actor.finishBlobPublish( + publishingId, + rootHash, + BigInt(archiveData.byteLength), + BigInt(sourceFiles.length), + ); if ("err" in res) { console.log(chalk.red("Error: ") + res.err); process.exit(1); } + progress("Done"); + logUpdate.done(); + console.log( chalk.green("Published ") + `${config.package.name}@${config.package.version}`, diff --git a/cli/declarations/main/main.did b/cli/declarations/main/main.did index 12b6e94c..13f9361d 100644 --- a/cli/declarations/main/main.did +++ b/cli/declarations/main/main.did @@ -272,15 +272,30 @@ type PackageChanges = prevDocsCoverage: float64; tests: TestsChanges; }; +type CreateCertificateResult = + record { + blob_hash: text; + method: text; + }; type Main = service { + _immutableObjectStorageBlobsAreLive: (hashBytesList: vec blob) -> + (vec bool) query; + _immutableObjectStorageBlobsToDelete: () -> (vec text) query; + _immutableObjectStorageConfirmBlobDeletion: (hashBytesList: vec blob) -> (); + _immutableObjectStorageCreateCertificate: (hash: text) -> + (CreateCertificateResult); + _immutableObjectStorageUpdateGatewayPrincipals: () -> (); addMaintainer: (packageName: PackageName, newMaintainer: principal) -> (Result_3); addOwner: (packageName: PackageName, newOwner: principal) -> (Result_3); backup: () -> (); computeHashesForExistingFiles: () -> (); + finishBlobPublish: (publishingId: PublishingId, blobHash: text, archiveSize: nat, fileCount: nat) -> (Result); finishPublish: (publishingId: PublishingId) -> (Result); getApiVersion: () -> (Text) query; + getBlobHash: (name: PackageName, version: PackageVersion) -> + (opt text) query; getBackupCanisterId: () -> (principal) query; getDefaultPackages: (dfxVersion: text) -> (vec record { @@ -350,6 +365,7 @@ type Main = setUserProp: (prop: text, value: text) -> (Result_3); startFileUpload: (publishingId: PublishingId, path: Text, chunkCount: nat, firstChunk: blob) -> (Result_2); + startBlobPublish: (configPub: PackageConfigV3_Publishing) -> (Result_1); startPublish: (configPub: PackageConfigV3_Publishing) -> (Result_1); takeSnapshotsIfNeeded: () -> (); transformRequest: (arg: TransformArg) -> (HttpRequestResult) query; diff --git a/cli/declarations/main/main.did.d.ts b/cli/declarations/main/main.did.d.ts index b83117c6..5692490a 100644 --- a/cli/declarations/main/main.did.d.ts +++ b/cli/declarations/main/main.did.d.ts @@ -45,13 +45,36 @@ export interface HttpRequestResult { 'body' : Uint8Array | number[], 'headers' : Array, } +export interface CreateCertificateResult { + 'method' : string, + 'blob_hash' : string, +} export interface Main { + '_immutableObjectStorageBlobsAreLive' : ActorMethod< + [Array], + Array + >, + '_immutableObjectStorageBlobsToDelete' : ActorMethod<[], Array>, + '_immutableObjectStorageConfirmBlobDeletion' : ActorMethod< + [Array], + undefined + >, + '_immutableObjectStorageCreateCertificate' : ActorMethod< + [string], + CreateCertificateResult + >, + '_immutableObjectStorageUpdateGatewayPrincipals' : ActorMethod< + [], + undefined + >, 'addMaintainer' : ActorMethod<[PackageName, Principal], Result_3>, 'addOwner' : ActorMethod<[PackageName, Principal], Result_3>, 'backup' : ActorMethod<[], undefined>, 'computeHashesForExistingFiles' : ActorMethod<[], undefined>, + 'finishBlobPublish' : ActorMethod<[PublishingId, string, bigint, bigint], Result>, 'finishPublish' : ActorMethod<[PublishingId], Result>, 'getApiVersion' : ActorMethod<[], Text>, + 'getBlobHash' : ActorMethod<[PackageName, PackageVersion], [] | [string]>, 'getBackupCanisterId' : ActorMethod<[], Principal>, 'getDefaultPackages' : ActorMethod< [string], @@ -122,6 +145,7 @@ export interface Main { [PublishingId, Text, bigint, Uint8Array | number[]], Result_2 >, + 'startBlobPublish' : ActorMethod<[PackageConfigV3_Publishing], Result_1>, 'startPublish' : ActorMethod<[PackageConfigV3_Publishing], Result_1>, 'takeSnapshotsIfNeeded' : ActorMethod<[], undefined>, 'transformRequest' : ActorMethod<[TransformArg], HttpRequestResult>, diff --git a/cli/declarations/main/main.did.js b/cli/declarations/main/main.did.js index 3acbbb06..23bcfcb3 100644 --- a/cli/declarations/main/main.did.js +++ b/cli/declarations/main/main.did.js @@ -259,13 +259,44 @@ export const idlFactory = ({ IDL }) => { 'context' : IDL.Vec(IDL.Nat8), 'response' : HttpRequestResult, }); + const CreateCertificateResult = IDL.Record({ + 'method' : IDL.Text, + 'blob_hash' : IDL.Text, + }); const Main = IDL.Service({ + '_immutableObjectStorageBlobsAreLive' : IDL.Func( + [IDL.Vec(IDL.Vec(IDL.Nat8))], + [IDL.Vec(IDL.Bool)], + ['query'], + ), + '_immutableObjectStorageBlobsToDelete' : IDL.Func( + [], + [IDL.Vec(IDL.Text)], + ['query'], + ), + '_immutableObjectStorageConfirmBlobDeletion' : IDL.Func( + [IDL.Vec(IDL.Vec(IDL.Nat8))], + [], + [], + ), + '_immutableObjectStorageCreateCertificate' : IDL.Func( + [IDL.Text], + [CreateCertificateResult], + [], + ), + '_immutableObjectStorageUpdateGatewayPrincipals' : IDL.Func([], [], []), 'addMaintainer' : IDL.Func([PackageName, IDL.Principal], [Result_3], []), 'addOwner' : IDL.Func([PackageName, IDL.Principal], [Result_3], []), 'backup' : IDL.Func([], [], []), 'computeHashesForExistingFiles' : IDL.Func([], [], []), + 'finishBlobPublish' : IDL.Func([PublishingId, IDL.Text, IDL.Nat, IDL.Nat], [Result], []), 'finishPublish' : IDL.Func([PublishingId], [Result], []), 'getApiVersion' : IDL.Func([], [Text], ['query']), + 'getBlobHash' : IDL.Func( + [PackageName, PackageVersion], + [IDL.Opt(IDL.Text)], + ['query'], + ), 'getBackupCanisterId' : IDL.Func([], [IDL.Principal], ['query']), 'getDefaultPackages' : IDL.Func( [IDL.Text], @@ -384,6 +415,7 @@ export const idlFactory = ({ IDL }) => { [Result_2], [], ), + 'startBlobPublish' : IDL.Func([PackageConfigV3_Publishing], [Result_1], []), 'startPublish' : IDL.Func([PackageConfigV3_Publishing], [Result_1], []), 'takeSnapshotsIfNeeded' : IDL.Func([], [], []), 'transformRequest' : IDL.Func( diff --git a/cli/integrity.ts b/cli/integrity.ts index 181f223d..56dec41a 100644 --- a/cli/integrity.ts +++ b/cli/integrity.ts @@ -31,7 +31,15 @@ type LockFileV3 = { deps: Record; }; -type LockFile = LockFileV1 | LockFileV2 | LockFileV3; +type LockFileV4 = { + version: 4; + mopsTomlDepsHash: string; + hashes: Record>; + blobHashes: Record; + deps: Record; +}; + +type LockFile = LockFileV1 | LockFileV2 | LockFileV3 | LockFileV4; export async function checkIntegrity(lock?: "check" | "update" | "ignore") { let force = !!lock; @@ -58,6 +66,27 @@ async function getFileHashesFromRegistry(): Promise< return fileHashesByPackageIds; } +async function getBlobHashesFromRegistry(): Promise> { + let packageIds = await getResolvedMopsPackageIds(); + let actor = await mainActor(); + let blobHashes: Record = {}; + + await Promise.all( + packageIds.map(async (packageId) => { + let [name, version] = packageId.split("@"); + if (!name || !version) { + return; + } + let result = await actor.getBlobHash(name, version); + if (result.length > 0 && result[0]) { + blobHashes[packageId] = result[0]; + } + }), + ); + + return blobHashes; +} + async function getResolvedMopsPackageIds(): Promise { let resolvedPackages = await resolvePackages(); let packageIds = Object.entries(resolvedPackages) @@ -149,7 +178,8 @@ export function checkLockFileLight(): boolean { if (existingLockFileJson) { let mopsTomlDepsHash = getMopsTomlDepsHash(); if ( - existingLockFileJson.version === 3 && + (existingLockFileJson.version === 3 || + existingLockFileJson.version === 4) && mopsTomlDepsHash === existingLockFileJson.mopsTomlDepsHash ) { return true; @@ -159,21 +189,28 @@ export function checkLockFileLight(): boolean { } export async function updateLockFile() { - // if lock file exists and mops.toml hasn't changed, don't update it if (checkLockFileLight()) { return; } let resolvedDeps = await resolvePackages(); - let fileHashes = await getFileHashesFromRegistry(); + let [fileHashes, blobHashes] = await Promise.all([ + getFileHashesFromRegistry(), + getBlobHashesFromRegistry(), + ]); + + let blobPackageIds = new Set(Object.keys(blobHashes)); - let lockFileJson: LockFileV3 = { - version: 3, + let lockFileJson: LockFileV4 = { + version: 4, mopsTomlDepsHash: getMopsTomlDepsHash(), deps: resolvedDeps, hashes: fileHashes.reduce( (acc, [packageId, fileHashes]) => { + if (blobPackageIds.has(packageId)) { + return acc; + } acc[packageId] = fileHashes.reduce( (acc, [fileId, hash]) => { acc[fileId] = bytesToHex(new Uint8Array(hash)); @@ -185,6 +222,7 @@ export async function updateLockFile() { }, {} as Record>, ), + blobHashes, }; let rootDir = getRootDir(); @@ -200,11 +238,10 @@ export async function updateLockFile() { // compare hashes of local files with hashes from the lock file export async function checkLockFile(force = false) { - let supportedVersions = [1, 2, 3]; + let supportedVersions = [1, 2, 3, 4]; let rootDir = getRootDir(); let lockFile = path.join(rootDir, "mops.lock"); - // check if lock file exists if (!fs.existsSync(lockFile)) { if (force) { console.error("Missing lock file. Run `mops install` to generate it."); @@ -218,7 +255,6 @@ export async function checkLockFile(force = false) { ); let packageIds = await getResolvedMopsPackageIds(); - // check lock file version if (!supportedVersions.includes(lockFileJsonGeneric.version)) { console.error("Integrity check failed"); console.error( @@ -240,8 +276,12 @@ export async function checkLockFile(force = false) { } } - // V2, V3: check mops.toml deps hash - if (lockFileJson.version === 2 || lockFileJson.version === 3) { + // V2, V3, V4: check mops.toml deps hash + if ( + lockFileJson.version === 2 || + lockFileJson.version === 3 || + lockFileJson.version === 4 + ) { if (lockFileJson.mopsTomlDepsHash !== getMopsTomlDepsHash()) { console.error("Integrity check failed"); console.error("Mismatched mops.toml dependencies hash"); @@ -251,8 +291,8 @@ export async function checkLockFile(force = false) { } } - // V3: check locked deps (including GitHub and local packages) - if (lockFileJson.version === 3) { + // V3, V4: check locked deps + if (lockFileJson.version === 3 || lockFileJson.version === 4) { let lockedDeps = { ...lockFileJson.deps }; let resolvedDeps = await resolvePackages(); @@ -267,26 +307,32 @@ export async function checkLockFile(force = false) { } } - // check number of packages - if (Object.keys(lockFileJson.hashes).length !== packageIds.length) { + // V4: count includes both hashes and blobHashes + let blobHashes = + lockFileJson.version === 4 + ? lockFileJson.blobHashes + : ({} as Record); + let totalLockedPackages = + Object.keys(lockFileJson.hashes).length + Object.keys(blobHashes).length; + + if (totalLockedPackages !== packageIds.length) { console.error("Integrity check failed"); console.error( - `Mismatched number of resolved packages: ${JSON.stringify(Object.keys(lockFileJson.hashes).length)} vs ${JSON.stringify(packageIds.length)}`, + `Mismatched number of resolved packages: ${totalLockedPackages} vs ${packageIds.length}`, ); process.exit(1); } - // check if resolved packages are in the lock file for (let packageId of packageIds) { - if (!(packageId in lockFileJson.hashes)) { + if (!(packageId in lockFileJson.hashes) && !(packageId in blobHashes)) { console.error("Integrity check failed"); console.error(`Missing package ${packageId} in lock file`); process.exit(1); } } + // check per-file hashes for legacy packages for (let [packageId, hashes] of Object.entries(lockFileJson.hashes)) { - // check if package is in resolved packages if (!packageIds.includes(packageId)) { console.error("Integrity check failed"); console.error( @@ -296,7 +342,6 @@ export async function checkLockFile(force = false) { } for (let [fileId, lockedHash] of Object.entries(hashes)) { - // check if file belongs to package if (!fileId.startsWith(packageId + "/")) { console.error("Integrity check failed"); console.error( @@ -305,7 +350,6 @@ export async function checkLockFile(force = false) { process.exit(1); } - // local file hash vs hash from lock file let localHash = getLocalFileHash(fileId); if (lockedHash !== localHash) { console.error("Integrity check failed"); @@ -316,4 +360,37 @@ export async function checkLockFile(force = false) { } } } + + // V4: verify blob hashes against the registry + if (Object.keys(blobHashes).length > 0) { + let actor = await mainActor(); + for (let [packageId, lockedBlobHash] of Object.entries(blobHashes)) { + if (!packageIds.includes(packageId)) { + console.error("Integrity check failed"); + console.error( + `Package ${packageId} in lock file but not in resolved packages`, + ); + process.exit(1); + } + + let [name, version] = packageId.split("@"); + if (name && version) { + let result = await actor.getBlobHash(name, version); + if (result.length === 0 || !result[0]) { + console.error("Integrity check failed"); + console.error( + `Package ${packageId} has blob hash in lock file but not in registry`, + ); + process.exit(1); + } + if (result[0] !== lockedBlobHash) { + console.error("Integrity check failed"); + console.error(`Mismatched blob hash for ${packageId}`); + console.error(`Locked hash: ${lockedBlobHash}`); + console.error(`Registry hash: ${result[0]}`); + process.exit(1); + } + } + } + } } diff --git a/cli/jest.config.js b/cli/jest.config.js index fc9bed8f..35a3a91c 100644 --- a/cli/jest.config.js +++ b/cli/jest.config.js @@ -10,5 +10,8 @@ export default { transform: { "^.+\\.tsx?$": ["ts-jest", { useESM: true }], }, + moduleNameMapper: { + "^(\\.{1,2}/.*)\\.js$": "$1", + }, testTimeout: 60000, }; diff --git a/cli/mops.ts b/cli/mops.ts index 8f26c657..f73562db 100644 --- a/cli/mops.ts +++ b/cli/mops.ts @@ -15,7 +15,7 @@ import { getPackageId } from "./helpers/get-package-id.js"; import { FILE_PATH_REGEX } from "./constants.js"; // (!) make changes in pair with backend -export let apiVersion = "1.3"; +export let apiVersion = "1.4"; export let globalConfigDir = ""; export let globalCacheDir = ""; diff --git a/cli/tests/e2e-blob-publish/README.md b/cli/tests/e2e-blob-publish/README.md new file mode 100644 index 00000000..380be7eb --- /dev/null +++ b/cli/tests/e2e-blob-publish/README.md @@ -0,0 +1,3 @@ +# __e2e-blob-test + +E2E test fixture for blob storage publish/install round-trip. diff --git a/cli/tests/e2e-blob-publish/mops.toml b/cli/tests/e2e-blob-publish/mops.toml new file mode 100644 index 00000000..313e9684 --- /dev/null +++ b/cli/tests/e2e-blob-publish/mops.toml @@ -0,0 +1,4 @@ +[package] +name = "__e2e-blob-test" +version = "0.0.0" +description = "E2E test package for blob storage" diff --git a/cli/tests/e2e-blob-publish/src/lib.mo b/cli/tests/e2e-blob-publish/src/lib.mo new file mode 100644 index 00000000..1adbab21 --- /dev/null +++ b/cli/tests/e2e-blob-publish/src/lib.mo @@ -0,0 +1,3 @@ +module { + public func hello() : Text { "world" }; +}; diff --git a/cli/tests/e2e-blob-storage.test.ts b/cli/tests/e2e-blob-storage.test.ts new file mode 100644 index 00000000..258c9e39 --- /dev/null +++ b/cli/tests/e2e-blob-storage.test.ts @@ -0,0 +1,182 @@ +import { describe, expect, test, jest, afterAll } from "@jest/globals"; +import fs from "node:fs"; +import path from "node:path"; +import os from "node:os"; +import { create as tarCreate } from "tar"; +import { Ed25519KeyIdentity } from "@icp-sdk/core/identity"; +import { uploadBlob, downloadBlob } from "../api/storageClient"; +import { cli } from "./helpers"; + +const E2E_ENABLED = Boolean(process.env.MOPS_TEST_E2E); +const E2E_PUBLISH_ENABLED = + E2E_ENABLED && Boolean(process.env.MOPS_IDENTITY_PEM); + +const describeE2E = E2E_ENABLED ? describe : describe.skip; +const describePublish = E2E_PUBLISH_ENABLED ? describe : describe.skip; + +jest.setTimeout(180_000); + +// Staging network -- all E2E tests target the staging registry so that IC +// response certificates are valid and the real Caffeine gateway accepts them. +const STAGING_ENV = { + MOPS_NETWORK: "staging", +}; + +function makeTmpDir(prefix: string): string { + return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); +} + +const tmpDirs: string[] = []; +afterAll(() => { + for (const d of tmpDirs) { + fs.rmSync(d, { recursive: true, force: true }); + } +}); + +async function createSmallTarGz(): Promise { + const dir = makeTmpDir("e2e-blob-upload-"); + tmpDirs.push(dir); + + fs.writeFileSync( + path.join(dir, "mops.toml"), + '[package]\nname = "dummy"\nversion = "0.0.1"\n', + ); + fs.writeFileSync(path.join(dir, "README.md"), "# dummy\n"); + fs.mkdirSync(path.join(dir, "src")); + fs.writeFileSync(path.join(dir, "src/lib.mo"), "module {};\n"); + + const archivePath = path.join(dir, "archive.tar.gz"); + await tarCreate({ gzip: true, file: archivePath, cwd: dir, portable: true }, [ + "mops.toml", + "README.md", + "src/lib.mo", + ]); + return new Uint8Array(fs.readFileSync(archivePath)); +} + +// --------------------------------------------------------------------------- +// Test A: Programmatic upload → download round-trip +// --------------------------------------------------------------------------- +describeE2E("E2E: blob upload/download round-trip", () => { + test("upload to gateway then download returns identical bytes", async () => { + const archiveData = await createSmallTarGz(); + const identity = Ed25519KeyIdentity.generate(); + + const rootHash = await uploadBlob(archiveData, identity); + expect(rootHash).toMatch(/^sha256:[0-9a-f]{64}$/); + + const downloaded = await downloadBlob(rootHash); + expect(Buffer.from(downloaded)).toEqual(Buffer.from(archiveData)); + }); + + test("uploading the same content twice returns the same hash (content-addressed)", async () => { + const archiveData = await createSmallTarGz(); + const identity = Ed25519KeyIdentity.generate(); + + const hash1 = await uploadBlob(archiveData, identity); + const hash2 = await uploadBlob(archiveData, identity); + expect(hash1).toBe(hash2); + }); + + test("onProgress callback is called during upload", async () => { + const archiveData = await createSmallTarGz(); + const identity = Ed25519KeyIdentity.generate(); + + const progressValues: number[] = []; + await uploadBlob(archiveData, identity, (pct) => progressValues.push(pct)); + + expect(progressValues.length).toBeGreaterThan(0); + expect(progressValues[progressValues.length - 1]).toBe(100); + }); +}); + +// --------------------------------------------------------------------------- +// Test B: Full CLI publish + install round-trip +// --------------------------------------------------------------------------- +describePublish("E2E: blob publish + install round-trip", () => { + const fixtureSrc = path.join(import.meta.dirname, "e2e-blob-publish"); + const uniqueVersion = `0.0.${Date.now() % 100000}`; + + let publishDir: string; + let installDir: string; + + test("publish a package via blob storage", async () => { + publishDir = makeTmpDir("e2e-blob-pub-"); + tmpDirs.push(publishDir); + + // Write identity PEM to the mops config dir + const mopsConfigDir = + process.platform === "darwin" + ? path.join(os.homedir(), "Library/Application Support/mops") + : path.join( + process.env.XDG_CONFIG_HOME || path.join(os.homedir(), ".config"), + "mops", + ); + fs.mkdirSync(mopsConfigDir, { recursive: true }); + const pemPath = path.join(mopsConfigDir, "identity.pem"); + const hadPem = fs.existsSync(pemPath); + let oldPem: Buffer | undefined; + if (hadPem) { + oldPem = fs.readFileSync(pemPath); + } + + try { + fs.writeFileSync(pemPath, process.env.MOPS_IDENTITY_PEM!); + + // Copy fixture and patch version + fs.cpSync(fixtureSrc, publishDir, { recursive: true }); + const toml = fs + .readFileSync(path.join(publishDir, "mops.toml"), "utf-8") + .replace('version = "0.0.0"', `version = "${uniqueVersion}"`); + fs.writeFileSync(path.join(publishDir, "mops.toml"), toml); + + const result = await cli(["publish"], { + cwd: publishDir, + env: STAGING_ENV, + }); + expect(result.exitCode).toBe(0); + expect(result.stdout + result.stderr).toMatch(/Published/i); + } finally { + if (hadPem && oldPem) { + fs.writeFileSync(pemPath, oldPem); + } else if (!hadPem) { + fs.rmSync(pemPath, { force: true }); + } + } + }); + + test("install the published package via blob storage", async () => { + installDir = makeTmpDir("e2e-blob-inst-"); + tmpDirs.push(installDir); + + fs.writeFileSync( + path.join(installDir, "mops.toml"), + `[package]\nname = "e2e-consumer"\nversion = "0.0.1"\n\n[dependencies]\n__e2e-blob-test = "${uniqueVersion}"\n`, + ); + + const result = await cli(["install"], { + cwd: installDir, + env: STAGING_ENV, + }); + expect(result.exitCode).toBe(0); + + // Verify files exist in .mops + const pkgDir = path.join( + installDir, + ".mops", + `__e2e-blob-test@${uniqueVersion}`, + ); + expect(fs.existsSync(path.join(pkgDir, "src/lib.mo"))).toBe(true); + expect(fs.existsSync(path.join(pkgDir, "README.md"))).toBe(true); + + // Verify lock file has blobHashes + const lockPath = path.join(installDir, "mops.lock"); + expect(fs.existsSync(lockPath)).toBe(true); + const lock = JSON.parse(fs.readFileSync(lockPath, "utf-8")); + expect(lock.version).toBe(4); + expect(lock.blobHashes).toBeDefined(); + expect(lock.blobHashes[`__e2e-blob-test@${uniqueVersion}`]).toMatch( + /^sha256:/, + ); + }); +}); diff --git a/cli/tests/storage-client.test.ts b/cli/tests/storage-client.test.ts new file mode 100644 index 00000000..b1d903c8 --- /dev/null +++ b/cli/tests/storage-client.test.ts @@ -0,0 +1,391 @@ +import { afterEach, describe, expect, test } from "@jest/globals"; +import { + buildMerkleTree, + splitChunks, + chunkHash, + nodeHash, + metadataHash, + hashToShaString, + getDownloadUrl, +} from "../api/storageClient"; + +const CHUNK_SIZE = 1024 * 1024; // 1 MiB — must match storageClient.ts + +function randomBytes(n: number): Uint8Array { + const buf = new Uint8Array(n); + for (let i = 0; i < n; i++) { + buf[i] = (i * 137 + 43) & 0xff; + } + return buf; +} + +describe("splitChunks", () => { + test("single byte produces one chunk", () => { + const chunks = splitChunks(new Uint8Array([42])); + expect(chunks).toHaveLength(1); + expect(chunks[0]).toEqual(new Uint8Array([42])); + }); + + test("exactly 1 MiB produces one chunk", () => { + const data = randomBytes(CHUNK_SIZE); + const chunks = splitChunks(data); + expect(chunks).toHaveLength(1); + expect(chunks[0]!.length).toBe(CHUNK_SIZE); + }); + + test("1 MiB + 1 byte produces two chunks", () => { + const data = randomBytes(CHUNK_SIZE + 1); + const chunks = splitChunks(data); + expect(chunks).toHaveLength(2); + expect(chunks[0]!.length).toBe(CHUNK_SIZE); + expect(chunks[1]!.length).toBe(1); + }); + + test("3 MiB exactly produces three chunks", () => { + const data = randomBytes(CHUNK_SIZE * 3); + const chunks = splitChunks(data); + expect(chunks).toHaveLength(3); + for (const c of chunks) { + expect(c.length).toBe(CHUNK_SIZE); + } + }); + + test("concatenated chunks equal original data", () => { + const data = randomBytes(CHUNK_SIZE * 2 + 777); + const chunks = splitChunks(data); + const reassembled = new Uint8Array(data.length); + let offset = 0; + for (const c of chunks) { + reassembled.set(c, offset); + offset += c.length; + } + expect(reassembled).toEqual(data); + }); +}); + +describe("chunkHash", () => { + test("is deterministic", () => { + const data = new Uint8Array([1, 2, 3]); + expect(chunkHash(data)).toEqual(chunkHash(data)); + }); + + test("different data produces different hash", () => { + const a = chunkHash(new Uint8Array([1])); + const b = chunkHash(new Uint8Array([2])); + expect(a).not.toEqual(b); + }); + + test("returns 32 bytes (SHA-256)", () => { + expect(chunkHash(new Uint8Array([0])).length).toBe(32); + }); +}); + +describe("nodeHash", () => { + test("is deterministic", () => { + const left = new Uint8Array(32).fill(0xaa); + const right = new Uint8Array(32).fill(0xbb); + expect(nodeHash(left, right)).toEqual(nodeHash(left, right)); + }); + + test("order matters", () => { + const left = new Uint8Array(32).fill(0xaa); + const right = new Uint8Array(32).fill(0xbb); + expect(nodeHash(left, right)).not.toEqual(nodeHash(right, left)); + }); + + test("null right (unbalanced) produces valid hash", () => { + const left = new Uint8Array(32).fill(0xcc); + const hash = nodeHash(left, null); + expect(hash.length).toBe(32); + }); + + test("null left (unbalanced) produces valid hash", () => { + const right = new Uint8Array(32).fill(0xdd); + const hash = nodeHash(null, right); + expect(hash.length).toBe(32); + }); + + test("null left != null right for same sibling", () => { + const child = new Uint8Array(32).fill(0xee); + expect(nodeHash(child, null)).not.toEqual(nodeHash(null, child)); + }); + + test("both null produces valid hash", () => { + const hash = nodeHash(null, null); + expect(hash.length).toBe(32); + }); + + test("returns 32 bytes (SHA-256)", () => { + const left = new Uint8Array(32).fill(1); + const right = new Uint8Array(32).fill(2); + expect(nodeHash(left, right).length).toBe(32); + }); +}); + +describe("metadataHash", () => { + test("is deterministic", () => { + const headers = { "Content-Type": "text/plain", "Content-Length": "42" }; + expect(metadataHash(headers)).toEqual(metadataHash(headers)); + }); + + test("header order does not matter (sorted internally)", () => { + const a = metadataHash({ + "Content-Length": "10", + "Content-Type": "text/plain", + }); + const b = metadataHash({ + "Content-Type": "text/plain", + "Content-Length": "10", + }); + expect(a).toEqual(b); + }); + + test("different values produce different hash", () => { + const a = metadataHash({ "Content-Type": "text/plain" }); + const b = metadataHash({ "Content-Type": "application/gzip" }); + expect(a).not.toEqual(b); + }); + + test("empty headers produces valid hash", () => { + const hash = metadataHash({}); + expect(hash.length).toBe(32); + }); + + test("returns 32 bytes (SHA-256)", () => { + expect(metadataHash({ "X-Test": "value" }).length).toBe(32); + }); +}); + +describe("hashToShaString", () => { + test("produces sha256: prefix with hex", () => { + const hash = new Uint8Array(32).fill(0); + const str = hashToShaString(hash); + expect(str).toMatch(/^sha256:[0-9a-f]{64}$/); + }); + + test("all-zero hash", () => { + const hash = new Uint8Array(32).fill(0); + expect(hashToShaString(hash)).toBe( + "sha256:0000000000000000000000000000000000000000000000000000000000000000", + ); + }); + + test("all-ff hash", () => { + const hash = new Uint8Array(32).fill(0xff); + expect(hashToShaString(hash)).toBe( + "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + ); + }); +}); + +describe("domain separation", () => { + test("chunkHash != nodeHash for same 32-byte input", () => { + const data = new Uint8Array(32).fill(0xab); + const ch = chunkHash(data); + const nh = nodeHash(data, null); + expect(ch).not.toEqual(nh); + }); + + test("chunkHash != metadataHash for overlapping content", () => { + const data = new TextEncoder().encode("Content-Type: text/plain\n"); + const ch = chunkHash(data); + const mh = metadataHash({ "Content-Type": "text/plain" }); + expect(ch).not.toEqual(mh); + }); +}); + +describe("buildMerkleTree", () => { + test("throws on empty data", () => { + expect(() => + buildMerkleTree(new Uint8Array(0), "application/gzip"), + ).toThrow("empty"); + }); + + test("single-chunk file", () => { + const data = randomBytes(100); + const result = buildMerkleTree(data, "application/gzip"); + + expect(result.chunks).toHaveLength(1); + expect(result.chunkHashes).toHaveLength(1); + expect(result.blobTree.tree_type).toBe("DSBMTWH"); + expect(result.blobTree.chunk_hashes).toHaveLength(1); + expect(result.rootHash).toMatch(/^sha256:[0-9a-f]{64}$/); + + // tree root has left (chunks subtree) and right (metadata) + expect(result.blobTree.tree.left).not.toBeNull(); + expect(result.blobTree.tree.right).not.toBeNull(); + }); + + test("multi-chunk file", () => { + const data = randomBytes(CHUNK_SIZE * 3 + 500); + const result = buildMerkleTree(data, "application/gzip"); + + expect(result.chunks).toHaveLength(4); + expect(result.chunkHashes).toHaveLength(4); + expect(result.blobTree.chunk_hashes).toHaveLength(4); + }); + + test("headers are sorted and contain Content-Type and Content-Length", () => { + const data = randomBytes(42); + const result = buildMerkleTree(data, "application/gzip"); + + expect(result.blobTree.headers).toHaveLength(2); + // sorted: Content-Length before Content-Type + expect(result.blobTree.headers[0]).toMatch(/^Content-Length: 42$/); + expect(result.blobTree.headers[1]).toMatch( + /^Content-Type: application\/gzip$/, + ); + }); + + test("determinism: same input produces same rootHash", () => { + const data = randomBytes(5000); + const a = buildMerkleTree(data, "application/gzip"); + const b = buildMerkleTree(data, "application/gzip"); + expect(a.rootHash).toBe(b.rootHash); + expect(a.blobTree.tree.hash).toBe(b.blobTree.tree.hash); + }); + + test("different data produces different rootHash", () => { + const a = buildMerkleTree(randomBytes(100), "application/gzip"); + const b = buildMerkleTree( + new Uint8Array(100).fill(0xff), + "application/gzip", + ); + expect(a.rootHash).not.toBe(b.rootHash); + }); + + test("different content type produces different rootHash", () => { + const data = randomBytes(100); + const a = buildMerkleTree(data, "application/gzip"); + const b = buildMerkleTree(data, "text/plain"); + expect(a.rootHash).not.toBe(b.rootHash); + }); + + test("chunk hashes in blobTree match chunkHashes array", () => { + const data = randomBytes(CHUNK_SIZE + 500); + const result = buildMerkleTree(data, "application/gzip"); + + for (let i = 0; i < result.chunkHashes.length; i++) { + expect(result.blobTree.chunk_hashes[i]).toBe( + hashToShaString(result.chunkHashes[i]!), + ); + } + }); + + test("each chunkHash matches chunkHash(chunk data)", () => { + const data = randomBytes(CHUNK_SIZE * 2 + 100); + const result = buildMerkleTree(data, "application/gzip"); + + for (let i = 0; i < result.chunks.length; i++) { + expect(result.chunkHashes[i]).toEqual(chunkHash(result.chunks[i]!)); + } + }); + + test("1-byte input (minimal case)", () => { + const data = new Uint8Array([0x42]); + const result = buildMerkleTree(data, "application/gzip"); + + expect(result.chunks).toHaveLength(1); + expect(result.chunks[0]).toEqual(data); + expect(result.blobTree.headers[0]).toMatch(/^Content-Length: 1$/); + expect(result.rootHash).toMatch(/^sha256:[0-9a-f]{64}$/); + }); + + test("2-chunk balanced tree has correct shape", () => { + const data = randomBytes(CHUNK_SIZE + 1); + const result = buildMerkleTree(data, "application/gzip"); + + expect(result.chunks).toHaveLength(2); + // chunks subtree root should have two children (left and right leaf) + const chunksSubtree = result.blobTree.tree.left!; + expect(chunksSubtree.left).not.toBeNull(); + expect(chunksSubtree.right).not.toBeNull(); + // both children are leaves (no further children) + expect(chunksSubtree.left!.left).toBeNull(); + expect(chunksSubtree.left!.right).toBeNull(); + expect(chunksSubtree.right!.left).toBeNull(); + expect(chunksSubtree.right!.right).toBeNull(); + }); + + test("3-chunk unbalanced tree has correct shape", () => { + const data = randomBytes(CHUNK_SIZE * 2 + 1); + const result = buildMerkleTree(data, "application/gzip"); + + expect(result.chunks).toHaveLength(3); + const chunksSubtree = result.blobTree.tree.left!; + expect(chunksSubtree.left).not.toBeNull(); + expect(chunksSubtree.right).not.toBeNull(); + // left child is a node pairing chunks 0 and 1 + expect(chunksSubtree.left!.left).not.toBeNull(); + expect(chunksSubtree.left!.right).not.toBeNull(); + // right child wraps chunk 2 with a null sibling (unbalanced) + expect(chunksSubtree.right!.left).not.toBeNull(); + expect(chunksSubtree.right!.right).toBeNull(); + // chunk 2 inside is a leaf + expect(chunksSubtree.right!.left!.left).toBeNull(); + expect(chunksSubtree.right!.left!.right).toBeNull(); + }); + + test("root hash is recomputable from tree components", () => { + const data = randomBytes(500); + const result = buildMerkleTree(data, "application/gzip"); + + // Recompute: rootHash = nodeHash(chunksRootHash, metadataRootHash) + const chunksRootHash = result.chunkHashes[0]!; + const metaHash = metadataHash({ + "Content-Type": "application/gzip", + "Content-Length": "500", + }); + const expectedRoot = nodeHash(chunksRootHash, metaHash); + expect(result.rootHash).toBe(hashToShaString(expectedRoot)); + }); + + test("single-chunk tree: chunks subtree is a leaf", () => { + const data = randomBytes(100); + const result = buildMerkleTree(data, "application/gzip"); + + // For 1 chunk, the chunks subtree is a leaf node (no children) + const chunksSubtree = result.blobTree.tree.left!; + expect(chunksSubtree.left).toBeNull(); + expect(chunksSubtree.right).toBeNull(); + // metadata node is always a leaf + const metaNode = result.blobTree.tree.right!; + expect(metaNode.left).toBeNull(); + expect(metaNode.right).toBeNull(); + }); +}); + +describe("getDownloadUrl", () => { + const origEnv = { ...process.env }; + + afterEach(() => { + process.env = { ...origEnv }; + }); + + test("includes blob_hash, owner_id, and project_id", () => { + process.env.MOPS_STORAGE_GATEWAY_URL = "https://test-gw.example.com"; + process.env.MOPS_STORAGE_PROJECT_ID = "test-project-123"; + process.env.MOPS_NETWORK = "ic"; + delete process.env.MOPS_REGISTRY_CANISTER_ID; + + const hash = + "sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890"; + const url = getDownloadUrl(hash); + + expect(url).toContain("https://test-gw.example.com/"); + expect(url).toContain(`blob_hash=${encodeURIComponent(hash)}`); + expect(url).toContain("owner_id="); + expect(url).toContain("project_id=test-project-123"); + }); + + test("uses default gateway when env var is unset", () => { + delete process.env.MOPS_STORAGE_GATEWAY_URL; + process.env.MOPS_NETWORK = "ic"; + delete process.env.MOPS_REGISTRY_CANISTER_ID; + + const url = getDownloadUrl( + "sha256:0000000000000000000000000000000000000000000000000000000000000000", + ); + expect(url).toContain("https://blob.caffeine.ai/"); + }); +}); diff --git a/cli/tests/tar-roundtrip.test.ts b/cli/tests/tar-roundtrip.test.ts new file mode 100644 index 00000000..571aeac0 --- /dev/null +++ b/cli/tests/tar-roundtrip.test.ts @@ -0,0 +1,217 @@ +import { describe, expect, test, afterAll } from "@jest/globals"; +import fs from "node:fs"; +import path from "node:path"; +import os from "node:os"; +import { create as tarCreate } from "tar"; +import { Parser as TarParser, type ReadEntry } from "tar"; + +const SAMPLE_FILES: Record = { + "mops.toml": '[package]\nname = "test-pkg"\nversion = "1.0.0"\n', + "README.md": "# test-pkg\nA test package.\n", + "src/lib.mo": 'module {\n public func hello() : Text { "world" };\n};\n', +}; + +let tmpDir: string; + +function setup(): string { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tar-roundtrip-")); + for (const [relPath, content] of Object.entries(SAMPLE_FILES)) { + const fullPath = path.join(tmpDir, relPath); + fs.mkdirSync(path.dirname(fullPath), { recursive: true }); + fs.writeFileSync(fullPath, content); + } + return tmpDir; +} + +afterAll(() => { + if (tmpDir) { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } +}); + +async function extractTarGz( + archiveBuffer: Uint8Array, +): Promise> { + const files = new Map(); + + await new Promise((resolve, reject) => { + const parser = new TarParser(); + parser.on("entry", (entry: ReadEntry) => { + const chunks: Buffer[] = []; + entry.on("data", (chunk: Buffer) => chunks.push(chunk)); + entry.on("end", () => { + const data = Buffer.concat(chunks); + files.set(entry.path, data.toString("utf-8")); + }); + }); + parser.on("end", resolve); + parser.on("error", reject); + parser.write(Buffer.from(archiveBuffer)); + parser.end(); + }); + + return files; +} + +async function extractTarGzRaw( + archiveBuffer: Uint8Array, +): Promise> { + const files = new Map(); + + await new Promise((resolve, reject) => { + const parser = new TarParser(); + parser.on("entry", (entry: ReadEntry) => { + const chunks: Buffer[] = []; + entry.on("data", (chunk: Buffer) => chunks.push(chunk)); + entry.on("end", () => { + files.set(entry.path, Buffer.concat(chunks)); + }); + }); + parser.on("end", resolve); + parser.on("error", reject); + parser.write(Buffer.from(archiveBuffer)); + parser.end(); + }); + + return files; +} + +describe("tar.gz round-trip (publish -> download contract)", () => { + test("archive and extract preserves file paths and contents", async () => { + const dir = setup(); + const archivePath = path.join(dir, "archive.tar.gz"); + + await tarCreate( + { + gzip: true, + file: archivePath, + cwd: dir, + portable: true, + }, + Object.keys(SAMPLE_FILES), + ); + + const archiveData = new Uint8Array(fs.readFileSync(archivePath)); + expect(archiveData.length).toBeGreaterThan(0); + + const extracted = await extractTarGz(archiveData); + + expect(extracted.size).toBe(Object.keys(SAMPLE_FILES).length); + + for (const [relPath, expectedContent] of Object.entries(SAMPLE_FILES)) { + expect(extracted.has(relPath)).toBe(true); + expect(extracted.get(relPath)).toBe(expectedContent); + } + }); + + test("nested directories are preserved", async () => { + const dir = setup(); + const archivePath = path.join(dir, "archive.tar.gz"); + + await tarCreate( + { gzip: true, file: archivePath, cwd: dir, portable: true }, + Object.keys(SAMPLE_FILES), + ); + + const archiveData = new Uint8Array(fs.readFileSync(archivePath)); + const extracted = await extractTarGz(archiveData); + + expect(extracted.has("src/lib.mo")).toBe(true); + }); + + test("archive is valid gzip (starts with gzip magic bytes)", async () => { + const dir = setup(); + const archivePath = path.join(dir, "archive.tar.gz"); + + await tarCreate( + { gzip: true, file: archivePath, cwd: dir, portable: true }, + Object.keys(SAMPLE_FILES), + ); + + const data = fs.readFileSync(archivePath); + expect(data[0]).toBe(0x1f); + expect(data[1]).toBe(0x8b); + }); + + test("binary content is preserved without corruption", async () => { + const dir = setup(); + + const binaryData = Buffer.alloc(512); + for (let i = 0; i < 512; i++) { + binaryData[i] = i & 0xff; + } + fs.writeFileSync(path.join(dir, "binary.bin"), binaryData); + + const archivePath = path.join(dir, "archive.tar.gz"); + await tarCreate( + { gzip: true, file: archivePath, cwd: dir, portable: true }, + ["binary.bin"], + ); + + const archiveData = new Uint8Array(fs.readFileSync(archivePath)); + const extracted = await extractTarGzRaw(archiveData); + + expect(extracted.has("binary.bin")).toBe(true); + const extractedBuf = extracted.get("binary.bin")!; + expect(extractedBuf.length).toBe(512); + for (let i = 0; i < 512; i++) { + expect(extractedBuf[i]).toBe(i & 0xff); + } + }); + + test("deeply nested paths are preserved", async () => { + const dir = setup(); + + const deepPath = "a/b/c/d/e/deep.mo"; + const fullPath = path.join(dir, deepPath); + fs.mkdirSync(path.dirname(fullPath), { recursive: true }); + fs.writeFileSync(fullPath, "module {};\n"); + + const archivePath = path.join(dir, "archive.tar.gz"); + await tarCreate( + { gzip: true, file: archivePath, cwd: dir, portable: true }, + [deepPath], + ); + + const archiveData = new Uint8Array(fs.readFileSync(archivePath)); + const extracted = await extractTarGz(archiveData); + + expect(extracted.has(deepPath)).toBe(true); + expect(extracted.get(deepPath)).toBe("module {};\n"); + }); + + test("empty file is preserved", async () => { + const dir = setup(); + + fs.writeFileSync(path.join(dir, "empty.txt"), ""); + + const archivePath = path.join(dir, "archive.tar.gz"); + await tarCreate( + { gzip: true, file: archivePath, cwd: dir, portable: true }, + ["empty.txt"], + ); + + const archiveData = new Uint8Array(fs.readFileSync(archivePath)); + const extracted = await extractTarGz(archiveData); + + expect(extracted.has("empty.txt")).toBe(true); + expect(extracted.get("empty.txt")).toBe(""); + }); + + test("UTF-8 content is preserved", async () => { + const dir = setup(); + const utf8Content = "Hello 世界 🌍 café ñ"; + fs.writeFileSync(path.join(dir, "utf8.txt"), utf8Content); + + const archivePath = path.join(dir, "archive.tar.gz"); + await tarCreate( + { gzip: true, file: archivePath, cwd: dir, portable: true }, + ["utf8.txt"], + ); + + const archiveData = new Uint8Array(fs.readFileSync(archivePath)); + const extracted = await extractTarGz(archiveData); + + expect(extracted.get("utf8.txt")).toBe(utf8Content); + }); +}); diff --git a/package-lock.json b/package-lock.json index 9cca6501..829ffafc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -183,24 +183,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/eslint": { - "version": "8.56.6", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "@types/estree": "*", - "@types/json-schema": "*" - } - }, - "node_modules/@types/estree": { - "version": "1.0.5", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true - }, "node_modules/@types/json-schema": { "version": "7.0.15", "dev": true,