From b4b1a73f00ef7a7c56087c1c5e47dfb11166b1fa Mon Sep 17 00:00:00 2001 From: 0x189D7997 <199489335+0x189D7997@users.noreply.github.com> Date: Thu, 20 Nov 2025 15:42:55 +0000 Subject: [PATCH 1/3] Use ETag header for manifests without hash --- build.gradle | 1 + .../util/FileSystemNetworkManager.java | 80 +++++++++++++++---- .../winplay02/gitcraft/util/RemoteHelper.java | 2 +- .../manifest/BaseMetadataProvider.java | 4 +- ...istoricMojangLauncherMetadataProvider.java | 2 +- .../skyrising/SkyrisingMetadataProvider.java | 6 +- .../MojangLauncherMetadataProvider.java | 2 +- 7 files changed, 75 insertions(+), 22 deletions(-) diff --git a/build.gradle b/build.gradle index d0affd74..e5587358 100644 --- a/build.gradle +++ b/build.gradle @@ -74,6 +74,7 @@ dependencies { implementation "net.fabricmc:access-widener:${access_widener_version}" implementation "net.fabricmc:mapping-io:${mappingio_version}" implementation "net.fabricmc:fabric-loom:${loom_version}" + libImplementation "net.fabricmc:fabric-loom:${loom_version}" implementation "net.fabricmc.unpick:unpick:${unpick_version}" implementation "net.fabricmc.unpick:unpick-format-utils:${unpick_version}" diff --git a/src/lib/java/com/github/winplay02/gitcraft/util/FileSystemNetworkManager.java b/src/lib/java/com/github/winplay02/gitcraft/util/FileSystemNetworkManager.java index ded68fd6..04c6262c 100644 --- a/src/lib/java/com/github/winplay02/gitcraft/util/FileSystemNetworkManager.java +++ b/src/lib/java/com/github/winplay02/gitcraft/util/FileSystemNetworkManager.java @@ -3,6 +3,7 @@ import com.github.winplay02.gitcraft.Library; import com.github.winplay02.gitcraft.integrity.IntegrityAlgorithm; import com.github.winplay02.gitcraft.pipeline.StepStatus; +import net.fabricmc.loom.util.AttributeHelper; import java.io.FileNotFoundException; import java.io.IOException; @@ -14,11 +15,13 @@ import java.net.http.HttpResponse; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.util.HashMap; import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; @@ -57,15 +60,15 @@ private static LockGuard acquireDownloadJobsReadLock() { protected static final Map completedJobs = new ConcurrentHashMap<>(); - public static CompletableFuture fetchRemoteSerialFSAccess(Executor executor, URI url, LocalFileInfo localFileInfo, boolean retry, boolean tolerateHashUnavailable) { + public static CompletableFuture fetchRemoteSerialFSAccess(Executor executor, URI url, LocalFileInfo localFileInfo, boolean retry, boolean tolerateHashUnavailable, boolean useEtag) { if (completedJobs.containsKey(localFileInfo.targetFile()) && - Objects.equals(completedJobs.get(localFileInfo.targetFile()).integrityChecksum, localFileInfo.checksum()) && - Objects.equals(completedJobs.get(localFileInfo.targetFile()).integrityAlgorithm, localFileInfo.integrityAlgorithm())) { + Objects.equals(completedJobs.get(localFileInfo.targetFile()).integrityChecksum(), localFileInfo.checksum()) && + Objects.equals(completedJobs.get(localFileInfo.targetFile()).integrityAlgorithm(), localFileInfo.integrityAlgorithm())) { return CompletableFuture.completedFuture(StepStatus.UP_TO_DATE); } if (completedJobs.containsKey(localFileInfo.targetFile()) && - !Objects.equals(completedJobs.get(localFileInfo.targetFile()).integrityChecksum, localFileInfo.checksum()) && - Objects.equals(completedJobs.get(localFileInfo.targetFile()).integrityAlgorithm, localFileInfo.integrityAlgorithm())) { + !Objects.equals(completedJobs.get(localFileInfo.targetFile()).integrityChecksum(), localFileInfo.checksum()) && + Objects.equals(completedJobs.get(localFileInfo.targetFile()).integrityAlgorithm(), localFileInfo.integrityAlgorithm())) { MiscHelper.panic("Cannot fulfill download to %s, there are multiple requests with different outcomes to the same file", localFileInfo.targetFile()); } try (LockGuard $ = acquireDownloadJobsWriteLock()) { @@ -83,7 +86,10 @@ public static CompletableFuture fetchRemoteSerialFSAccess(Executor e try { MiscHelper.println("Fetching %s %s from: %s", localFileInfo.outputFileKind(), localFileInfo.outputFileId(), url); try { - FileSystemNetworkManager.fetchFileAsync(url, localFileInfo.targetFile()).get(); + HttpResponse response = FileSystemNetworkManager.fetchFileAsync(url, localFileInfo.targetFile(), useEtag).get(); + if (useEtag && response.statusCode() == 304) { + MiscHelper.println("Skipped downloading %s %s as it is already up-to-date", localFileInfo.outputFileKind(), localFileInfo.outputFileId()); + } if (!retry) { break; } @@ -119,8 +125,19 @@ public static CompletableFuture fetchRemoteSerialFSAccess(Executor e protected static final Map connectionLimiter = new ConcurrentHashMap<>(); - protected static CompletableFuture> fetchFileAsync(URI uri, Path targetFile) { - HttpRequest request = HttpRequest.newBuilder(uri).GET().build(); + protected static CompletableFuture> fetchFileAsync(URI uri, Path targetFile, boolean useEtag) { + HttpRequest.Builder builder = HttpRequest.newBuilder(uri).GET(); + if (useEtag) { + try { + Optional etag = AttributeHelper.readAttribute(targetFile, "ETag"); + if (etag.isPresent()) { + builder.header("If-None-Match", etag.orElseThrow()); + } + } catch (IOException e) { + MiscHelper.panicBecause(e, "Cannot read etag for %s", targetFile); + } + } + HttpRequest request = builder.build(); final Semaphore semaphore = connectionLimiter.computeIfAbsent(uri.getHost().toLowerCase(Locale.ROOT), $ -> new Semaphore(Library.CONF_GLOBAL.maxConcurrentHttpRequestsPerOrigin())); if (targetFile.getParent() != null) { try { @@ -130,12 +147,47 @@ protected static CompletableFuture> fetchFileAsync(URI uri, P } } semaphore.acquireUninterruptibly(); - return httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofFile(targetFile, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE, StandardOpenOption.WRITE)).thenApply(response -> { - if (response.statusCode() == 404) { - MiscHelper.throwUnchecked(new FileNotFoundException(uri.toString())); - } - return response; - }).whenComplete(($, $$) -> semaphore.release()); + if (useEtag) { + Path tmpDownloadFile = targetFile.resolveSibling(targetFile.getFileName() + ".tmp"); // we have to download to temporary location because empty 304 Not Modified response erases file contents + return httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofFile(tmpDownloadFile, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE, StandardOpenOption.WRITE)).thenApply(response -> { + if (response.statusCode() >= 200 && response.statusCode() < 300) { + try { + Files.move(tmpDownloadFile, targetFile, StandardCopyOption.REPLACE_EXISTING); + + Optional received_etag = response.headers().firstValue("etag"); + if (received_etag.isPresent()) { + AttributeHelper.writeAttribute(targetFile, "ETag", received_etag.orElseThrow()); + } + } catch (IOException e) { + try { + Files.deleteIfExists(tmpDownloadFile); + } catch (IOException e1) { + MiscHelper.throwUnchecked(e1); + } + + MiscHelper.throwUnchecked(e); + } + } + + try { + Files.deleteIfExists(tmpDownloadFile); + } catch (IOException e) { + MiscHelper.throwUnchecked(e); + } + + if (response.statusCode() == 404) { + MiscHelper.throwUnchecked(new FileNotFoundException(uri.toString())); + } + return response; + }).whenComplete(($, $$) -> semaphore.release()); + } else { + return httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofFile(targetFile, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE, StandardOpenOption.WRITE)).thenApply(response -> { + if (response.statusCode() == 404) { + MiscHelper.throwUnchecked(new FileNotFoundException(uri.toString())); + } + return response; + }).whenComplete(($, $$) -> semaphore.release()); + } } public static String fetchAllFromURLSync(URL url) throws IOException, URISyntaxException, InterruptedException { diff --git a/src/lib/java/com/github/winplay02/gitcraft/util/RemoteHelper.java b/src/lib/java/com/github/winplay02/gitcraft/util/RemoteHelper.java index ccc57875..40208e8d 100644 --- a/src/lib/java/com/github/winplay02/gitcraft/util/RemoteHelper.java +++ b/src/lib/java/com/github/winplay02/gitcraft/util/RemoteHelper.java @@ -40,7 +40,7 @@ public static StepStatus downloadToFileWithChecksumIfNotExists(Executor executor } while (true) { try { - return FileSystemNetworkManager.fetchRemoteSerialFSAccess(executor, uri, localFileInfo, retry, false).get(); + return FileSystemNetworkManager.fetchRemoteSerialFSAccess(executor, uri, localFileInfo, retry, false, false).get(); } catch (InterruptedException e) { MiscHelper.println("Interrupted while waiting for download of %s to complete", url); continue; diff --git a/src/main/groovy/com/github/winplay02/gitcraft/manifest/BaseMetadataProvider.java b/src/main/groovy/com/github/winplay02/gitcraft/manifest/BaseMetadataProvider.java index 25211acd..2dddd13e 100644 --- a/src/main/groovy/com/github/winplay02/gitcraft/manifest/BaseMetadataProvider.java +++ b/src/main/groovy/com/github/winplay02/gitcraft/manifest/BaseMetadataProvider.java @@ -203,7 +203,7 @@ protected final CompletableFuture fetchVersionMetadata(Executor executor, String fileNameWithoutExt = fileName.substring(0, fileName.lastIndexOf('.')); String fileExt = fileName.lastIndexOf(".") != -1 ? fileName.substring(fileName.lastIndexOf(".") + 1) : ""; Path filePath = sha1 != null ? targetDir.resolve(String.format("%s_%s.%s", fileNameWithoutExt, sha1, fileExt)) : targetDir.resolve(fileName); // allow multiple files with same hash to coexist (in case of reuploads with same meta name, referenced from different versions) - CompletableFuture status = FileSystemNetworkManager.fetchRemoteSerialFSAccess(executor, uri, new FileSystemNetworkManager.LocalFileInfo(filePath, sha1, sha1 != null ? Library.IA_SHA1 : null, targetFileKind, id), true, false); + CompletableFuture status = FileSystemNetworkManager.fetchRemoteSerialFSAccess(executor, uri, new FileSystemNetworkManager.LocalFileInfo(filePath, sha1, sha1 != null ? Library.IA_SHA1 : null, targetFileKind, id), true, true); return status.thenApply($ -> { try { return this.loadVersionMetadata(filePath, metadataClass, fileName); @@ -222,7 +222,7 @@ protected final CompletableFuture fetchVersionMetadataFilename(Executor e throw new IOException(e); } Path filePath = targetDir.resolve(filename); - CompletableFuture status = FileSystemNetworkManager.fetchRemoteSerialFSAccess(executor, uri, new FileSystemNetworkManager.LocalFileInfo(filePath, sha1, sha1 != null ? Library.IA_SHA1 : null, targetFileKind, id), true, false); + CompletableFuture status = FileSystemNetworkManager.fetchRemoteSerialFSAccess(executor, uri, new FileSystemNetworkManager.LocalFileInfo(filePath, sha1, sha1 != null ? Library.IA_SHA1 : null, targetFileKind, id), true, true); return status.thenApply($ -> { try { return this.loadVersionMetadata(filePath, metadataClass, filePath.getFileName().toString()); diff --git a/src/main/groovy/com/github/winplay02/gitcraft/manifest/historic/HistoricMojangLauncherMetadataProvider.java b/src/main/groovy/com/github/winplay02/gitcraft/manifest/historic/HistoricMojangLauncherMetadataProvider.java index a28c65cb..96fe6236 100644 --- a/src/main/groovy/com/github/winplay02/gitcraft/manifest/historic/HistoricMojangLauncherMetadataProvider.java +++ b/src/main/groovy/com/github/winplay02/gitcraft/manifest/historic/HistoricMojangLauncherMetadataProvider.java @@ -237,7 +237,7 @@ protected void loadVersions(Executor executor) throws IOException { @Override protected CompletableFuture loadVersionFromManifest(Executor executor, MojangLauncherManifest.VersionEntry manifestEntry, Path targetDir) throws IOException { - CompletableFuture futureInfo = this.fetchVersionMetadata(executor, manifestEntry.id(), manifestEntry.url(), manifestEntry.sha1(), targetDir, "version info", VersionInfo.class); + CompletableFuture futureInfo = this.fetchVersionMetadata(executor, manifestEntry.id(), manifestEntry.url(), manifestEntry.sha1(), targetDir, "version info", VersionInfo.class, false); return futureInfo.thenApply(info -> { info = info.withUpdatedId(manifestEntry.id()); String semanticVersion = this.mojangLauncherMetadataProvider.lookupSemanticVersion(executor, info); diff --git a/src/main/groovy/com/github/winplay02/gitcraft/manifest/skyrising/SkyrisingMetadataProvider.java b/src/main/groovy/com/github/winplay02/gitcraft/manifest/skyrising/SkyrisingMetadataProvider.java index 9b513bb5..d9ac465d 100644 --- a/src/main/groovy/com/github/winplay02/gitcraft/manifest/skyrising/SkyrisingMetadataProvider.java +++ b/src/main/groovy/com/github/winplay02/gitcraft/manifest/skyrising/SkyrisingMetadataProvider.java @@ -58,7 +58,7 @@ protected void postLoadVersions() { public CompletableFuture fetchSpecificManifest(Executor executor, String id, VersionDetails.ManifestEntry manifestEntryPtr) { try { - return this.fetchVersionMetadataFilename(executor, String.format("%s_%s.json", id, manifestEntryPtr.hash()), id, manifestEntryPtr.url(), manifestEntryPtr.hash(), this.manifestMetadata.resolve("manifests"), "version manifest", VersionInfo.class); + return this.fetchVersionMetadataFilename(executor, String.format("%s_%s.json", id, manifestEntryPtr.hash()), id, manifestEntryPtr.url(), manifestEntryPtr.hash(), this.manifestMetadata.resolve("manifests"), "version manifest", VersionInfo.class, true); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -66,8 +66,8 @@ public CompletableFuture fetchSpecificManifest(Executor executor, S @Override protected CompletableFuture loadVersionFromManifest(Executor executor, SkyrisingManifest.VersionEntry manifestEntry, Path targetDir) throws IOException { - CompletableFuture infoFuture = this.fetchVersionMetadata(executor, manifestEntry.id(), manifestEntry.url(), null, targetDir.resolve("info"), "version info", VersionInfo.class); - CompletableFuture detailsFuture = this.fetchVersionMetadata(executor, manifestEntry.id(), manifestEntry.details(), null, targetDir.resolve("details"), "version details", VersionDetails.class); + CompletableFuture infoFuture = this.fetchVersionMetadata(executor, manifestEntry.id(), manifestEntry.url(), null, targetDir.resolve("info"), "version info", VersionInfo.class, true); + CompletableFuture detailsFuture = this.fetchVersionMetadata(executor, manifestEntry.id(), manifestEntry.details(), null, targetDir.resolve("details"), "version details", VersionDetails.class, true); return CompletableFuture.allOf(infoFuture, detailsFuture).thenApply($ -> { VersionInfo info = infoFuture.join(); VersionDetails details = detailsFuture.join(); diff --git a/src/main/groovy/com/github/winplay02/gitcraft/manifest/vanilla/MojangLauncherMetadataProvider.java b/src/main/groovy/com/github/winplay02/gitcraft/manifest/vanilla/MojangLauncherMetadataProvider.java index 2c5f081d..1ee37c30 100644 --- a/src/main/groovy/com/github/winplay02/gitcraft/manifest/vanilla/MojangLauncherMetadataProvider.java +++ b/src/main/groovy/com/github/winplay02/gitcraft/manifest/vanilla/MojangLauncherMetadataProvider.java @@ -167,7 +167,7 @@ public String getInternalName() { @Override protected CompletableFuture loadVersionFromManifest(Executor executor, MojangLauncherManifest.VersionEntry manifestEntry, Path targetDir) throws IOException { - CompletableFuture futureInfo = this.fetchVersionMetadata(executor, manifestEntry.id(), manifestEntry.url(), manifestEntry.sha1(), targetDir, "version info", VersionInfo.class); + CompletableFuture futureInfo = this.fetchVersionMetadata(executor, manifestEntry.id(), manifestEntry.url(), manifestEntry.sha1(), targetDir, "version info", VersionInfo.class, false); return futureInfo.thenApply(info -> { String semanticVersion = this.lookupSemanticVersion(executor, info); return OrderedVersion.from(info, semanticVersion); From d8ae45a3193957ec0970e350c85e362a0a9f99c4 Mon Sep 17 00:00:00 2001 From: 0x189D7997 <199489335+0x189D7997@users.noreply.github.com> Date: Thu, 20 Nov 2025 15:58:40 +0000 Subject: [PATCH 2/3] Fix mistake --- .../winplay02/gitcraft/manifest/BaseMetadataProvider.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/groovy/com/github/winplay02/gitcraft/manifest/BaseMetadataProvider.java b/src/main/groovy/com/github/winplay02/gitcraft/manifest/BaseMetadataProvider.java index 2dddd13e..b1f19882 100644 --- a/src/main/groovy/com/github/winplay02/gitcraft/manifest/BaseMetadataProvider.java +++ b/src/main/groovy/com/github/winplay02/gitcraft/manifest/BaseMetadataProvider.java @@ -192,7 +192,7 @@ private final M fetchVersionsManifest(MetadataSources.RemoteVersionsManifest loader) throws IOException; - protected final CompletableFuture fetchVersionMetadata(Executor executor, String id, String url, String sha1, Path targetDir, String targetFileKind, Class metadataClass) throws IOException { + protected final CompletableFuture fetchVersionMetadata(Executor executor, String id, String url, String sha1, Path targetDir, String targetFileKind, Class metadataClass, boolean useEtag) throws IOException { URI uri = null; try { uri = new URI(url); @@ -203,7 +203,7 @@ protected final CompletableFuture fetchVersionMetadata(Executor executor, String fileNameWithoutExt = fileName.substring(0, fileName.lastIndexOf('.')); String fileExt = fileName.lastIndexOf(".") != -1 ? fileName.substring(fileName.lastIndexOf(".") + 1) : ""; Path filePath = sha1 != null ? targetDir.resolve(String.format("%s_%s.%s", fileNameWithoutExt, sha1, fileExt)) : targetDir.resolve(fileName); // allow multiple files with same hash to coexist (in case of reuploads with same meta name, referenced from different versions) - CompletableFuture status = FileSystemNetworkManager.fetchRemoteSerialFSAccess(executor, uri, new FileSystemNetworkManager.LocalFileInfo(filePath, sha1, sha1 != null ? Library.IA_SHA1 : null, targetFileKind, id), true, true); + CompletableFuture status = FileSystemNetworkManager.fetchRemoteSerialFSAccess(executor, uri, new FileSystemNetworkManager.LocalFileInfo(filePath, sha1, sha1 != null ? Library.IA_SHA1 : null, targetFileKind, id), true, false, useEtag); return status.thenApply($ -> { try { return this.loadVersionMetadata(filePath, metadataClass, fileName); @@ -214,7 +214,7 @@ protected final CompletableFuture fetchVersionMetadata(Executor executor, }); } - protected final CompletableFuture fetchVersionMetadataFilename(Executor executor, String filename, String id, String url, String sha1, Path targetDir, String targetFileKind, Class metadataClass) throws IOException { + protected final CompletableFuture fetchVersionMetadataFilename(Executor executor, String filename, String id, String url, String sha1, Path targetDir, String targetFileKind, Class metadataClass, boolean useEtag) throws IOException { URI uri = null; try { uri = new URI(url); @@ -222,7 +222,7 @@ protected final CompletableFuture fetchVersionMetadataFilename(Executor e throw new IOException(e); } Path filePath = targetDir.resolve(filename); - CompletableFuture status = FileSystemNetworkManager.fetchRemoteSerialFSAccess(executor, uri, new FileSystemNetworkManager.LocalFileInfo(filePath, sha1, sha1 != null ? Library.IA_SHA1 : null, targetFileKind, id), true, true); + CompletableFuture status = FileSystemNetworkManager.fetchRemoteSerialFSAccess(executor, uri, new FileSystemNetworkManager.LocalFileInfo(filePath, sha1, sha1 != null ? Library.IA_SHA1 : null, targetFileKind, id), true, false, useEtag); return status.thenApply($ -> { try { return this.loadVersionMetadata(filePath, metadataClass, filePath.getFileName().toString()); From 1256769d3a3b310fd98f86e24491dc74094477c2 Mon Sep 17 00:00:00 2001 From: 0x189D7997 <199489335+0x189D7997@users.noreply.github.com> Date: Thu, 20 Nov 2025 20:08:53 +0000 Subject: [PATCH 3/3] Un-inline constant --- .../winplay02/gitcraft/util/FileSystemNetworkManager.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/lib/java/com/github/winplay02/gitcraft/util/FileSystemNetworkManager.java b/src/lib/java/com/github/winplay02/gitcraft/util/FileSystemNetworkManager.java index 04c6262c..064e77b0 100644 --- a/src/lib/java/com/github/winplay02/gitcraft/util/FileSystemNetworkManager.java +++ b/src/lib/java/com/github/winplay02/gitcraft/util/FileSystemNetworkManager.java @@ -124,12 +124,14 @@ public static CompletableFuture fetchRemoteSerialFSAccess(Executor e protected static final HttpClient httpClient = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NORMAL).build(); protected static final Map connectionLimiter = new ConcurrentHashMap<>(); - + + private static final String ETAG_ATTRIBUTE = "ETag"; + protected static CompletableFuture> fetchFileAsync(URI uri, Path targetFile, boolean useEtag) { HttpRequest.Builder builder = HttpRequest.newBuilder(uri).GET(); if (useEtag) { try { - Optional etag = AttributeHelper.readAttribute(targetFile, "ETag"); + Optional etag = AttributeHelper.readAttribute(targetFile, ETAG_ATTRIBUTE); if (etag.isPresent()) { builder.header("If-None-Match", etag.orElseThrow()); } @@ -156,7 +158,7 @@ protected static CompletableFuture> fetchFileAsync(URI uri, P Optional received_etag = response.headers().firstValue("etag"); if (received_etag.isPresent()) { - AttributeHelper.writeAttribute(targetFile, "ETag", received_etag.orElseThrow()); + AttributeHelper.writeAttribute(targetFile, ETAG_ATTRIBUTE, received_etag.orElseThrow()); } } catch (IOException e) { try {