diff --git a/build.gradle b/build.gradle index ba6b0c3..f7f104f 100644 --- a/build.gradle +++ b/build.gradle @@ -74,6 +74,7 @@ dependencies { implementation "net.fabricmc:access-widener:${access_widener_version}" implementation "net.fabricmc:mapping-io:${mappingio_version}" implementation "net.fabricmc:fabric-loom:${loom_version}" + libImplementation "net.fabricmc:fabric-loom:${loom_version}" implementation "net.fabricmc.unpick:unpick:${unpick_version}" implementation "net.fabricmc.unpick:unpick-format-utils:${unpick_version}" diff --git a/src/lib/java/com/github/winplay02/gitcraft/util/FileSystemNetworkManager.java b/src/lib/java/com/github/winplay02/gitcraft/util/FileSystemNetworkManager.java index c9e8e6e..ebc191f 100644 --- a/src/lib/java/com/github/winplay02/gitcraft/util/FileSystemNetworkManager.java +++ b/src/lib/java/com/github/winplay02/gitcraft/util/FileSystemNetworkManager.java @@ -3,6 +3,7 @@ import com.github.winplay02.gitcraft.Library; import com.github.winplay02.gitcraft.integrity.IntegrityAlgorithm; import com.github.winplay02.gitcraft.pipeline.StepStatus; +import net.fabricmc.loom.util.AttributeHelper; import java.io.FileNotFoundException; import java.io.IOException; @@ -14,11 +15,13 @@ import java.net.http.HttpResponse; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.util.HashMap; import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; @@ -57,15 +60,15 @@ private static LockGuard acquireDownloadJobsReadLock() { protected static final Map completedJobs = new ConcurrentHashMap<>(); - public static CompletableFuture fetchRemoteSerialFSAccess(Executor executor, URI url, LocalFileInfo localFileInfo, boolean retry, boolean tolerateHashUnavailable, int concurrentLimit) { + public static CompletableFuture fetchRemoteSerialFSAccess(Executor executor, URI url, LocalFileInfo localFileInfo, boolean retry, boolean tolerateHashUnavailable, int concurrentLimit, boolean useEtag) { if (completedJobs.containsKey(localFileInfo.targetFile()) && - Objects.equals(completedJobs.get(localFileInfo.targetFile()).integrityChecksum, localFileInfo.checksum()) && - Objects.equals(completedJobs.get(localFileInfo.targetFile()).integrityAlgorithm, localFileInfo.integrityAlgorithm())) { + Objects.equals(completedJobs.get(localFileInfo.targetFile()).integrityChecksum(), localFileInfo.checksum()) && + Objects.equals(completedJobs.get(localFileInfo.targetFile()).integrityAlgorithm(), localFileInfo.integrityAlgorithm())) { return CompletableFuture.completedFuture(StepStatus.UP_TO_DATE); } if (completedJobs.containsKey(localFileInfo.targetFile()) && - !Objects.equals(completedJobs.get(localFileInfo.targetFile()).integrityChecksum, localFileInfo.checksum()) && - Objects.equals(completedJobs.get(localFileInfo.targetFile()).integrityAlgorithm, localFileInfo.integrityAlgorithm())) { + !Objects.equals(completedJobs.get(localFileInfo.targetFile()).integrityChecksum(), localFileInfo.checksum()) && + Objects.equals(completedJobs.get(localFileInfo.targetFile()).integrityAlgorithm(), localFileInfo.integrityAlgorithm())) { MiscHelper.panic("Cannot fulfill download to %s, there are multiple requests with different outcomes to the same file", localFileInfo.targetFile()); } try (LockGuard $ = acquireDownloadJobsWriteLock()) { @@ -83,7 +86,10 @@ public static CompletableFuture fetchRemoteSerialFSAccess(Executor e try { MiscHelper.println("Fetching %s %s from: %s", localFileInfo.outputFileKind(), localFileInfo.outputFileId(), url); try { - FileSystemNetworkManager.fetchFileAsync(url, localFileInfo.targetFile(), concurrentLimit).get(); + HttpResponse response = FileSystemNetworkManager.fetchFileAsync(url, localFileInfo.targetFile(), concurrentLimit, useEtag).get(); + if (useEtag && response.statusCode() == 304) { + MiscHelper.println("Skipped downloading %s %s as it is already up-to-date", localFileInfo.outputFileKind(), localFileInfo.outputFileId()); + } if (!retry) { break; } @@ -118,9 +124,22 @@ public static CompletableFuture fetchRemoteSerialFSAccess(Executor e protected static final HttpClient httpClient = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NORMAL).build(); protected static final Map connectionLimiter = new ConcurrentHashMap<>(); - - protected static CompletableFuture> fetchFileAsync(URI uri, Path targetFile, int concurrentLimit) { - HttpRequest request = HttpRequest.newBuilder(uri).GET().build(); + + private static final String ETAG_ATTRIBUTE = "ETag"; + + protected static CompletableFuture> fetchFileAsync(URI uri, Path targetFile, int concurrentLimit, boolean useEtag) { + HttpRequest.Builder builder = HttpRequest.newBuilder(uri).GET(); + if (useEtag) { + try { + Optional etag = AttributeHelper.readAttribute(targetFile, ETAG_ATTRIBUTE); + if (etag.isPresent()) { + builder.header("If-None-Match", etag.orElseThrow()); + } + } catch (IOException e) { + MiscHelper.panicBecause(e, "Cannot read etag for %s", targetFile); + } + } + HttpRequest request = builder.build(); final Semaphore semaphore = connectionLimiter.computeIfAbsent(uri.getHost().toLowerCase(Locale.ROOT), $ -> new Semaphore(concurrentLimit > 0 ? Math.min(concurrentLimit, Library.CONF_GLOBAL.maxConcurrentHttpRequestsPerOrigin()) @@ -133,12 +152,47 @@ protected static CompletableFuture> fetchFileAsync(URI uri, P } } semaphore.acquireUninterruptibly(); - return httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofFile(targetFile, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE, StandardOpenOption.WRITE)).thenApply(response -> { - if (response.statusCode() == 404) { - MiscHelper.throwUnchecked(new FileNotFoundException(uri.toString())); - } - return response; - }).whenComplete(($, $$) -> semaphore.release()); + if (useEtag) { + Path tmpDownloadFile = targetFile.resolveSibling(targetFile.getFileName() + ".tmp"); // we have to download to temporary location because empty 304 Not Modified response erases file contents + return httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofFile(tmpDownloadFile, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE, StandardOpenOption.WRITE)).thenApply(response -> { + if (response.statusCode() >= 200 && response.statusCode() < 300) { + try { + Files.move(tmpDownloadFile, targetFile, StandardCopyOption.REPLACE_EXISTING); + + Optional received_etag = response.headers().firstValue("etag"); + if (received_etag.isPresent()) { + AttributeHelper.writeAttribute(targetFile, ETAG_ATTRIBUTE, received_etag.orElseThrow()); + } + } catch (IOException e) { + try { + Files.deleteIfExists(tmpDownloadFile); + } catch (IOException e1) { + MiscHelper.throwUnchecked(e1); + } + + MiscHelper.throwUnchecked(e); + } + } + + try { + Files.deleteIfExists(tmpDownloadFile); + } catch (IOException e) { + MiscHelper.throwUnchecked(e); + } + + if (response.statusCode() == 404) { + MiscHelper.throwUnchecked(new FileNotFoundException(uri.toString())); + } + return response; + }).whenComplete(($, $$) -> semaphore.release()); + } else { + return httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofFile(targetFile, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE, StandardOpenOption.WRITE)).thenApply(response -> { + if (response.statusCode() == 404) { + MiscHelper.throwUnchecked(new FileNotFoundException(uri.toString())); + } + return response; + }).whenComplete(($, $$) -> semaphore.release()); + } } public static String fetchAllFromURLSync(URL url) throws IOException, URISyntaxException, InterruptedException { diff --git a/src/lib/java/com/github/winplay02/gitcraft/util/RemoteHelper.java b/src/lib/java/com/github/winplay02/gitcraft/util/RemoteHelper.java index c7a5675..b24107e 100644 --- a/src/lib/java/com/github/winplay02/gitcraft/util/RemoteHelper.java +++ b/src/lib/java/com/github/winplay02/gitcraft/util/RemoteHelper.java @@ -40,7 +40,7 @@ public static StepStatus downloadToFileWithChecksumIfNotExists(Executor executor } while (true) { try { - return FileSystemNetworkManager.fetchRemoteSerialFSAccess(executor, uri, localFileInfo, retry, false, -1).get(); + return FileSystemNetworkManager.fetchRemoteSerialFSAccess(executor, uri, localFileInfo, retry, false, -1, false).get(); } catch (InterruptedException e) { MiscHelper.println("Interrupted while waiting for download of %s to complete", url); continue; diff --git a/src/main/groovy/com/github/winplay02/gitcraft/manifest/BaseMetadataProvider.java b/src/main/groovy/com/github/winplay02/gitcraft/manifest/BaseMetadataProvider.java index fdf8431..c8f467b 100644 --- a/src/main/groovy/com/github/winplay02/gitcraft/manifest/BaseMetadataProvider.java +++ b/src/main/groovy/com/github/winplay02/gitcraft/manifest/BaseMetadataProvider.java @@ -215,7 +215,7 @@ public int getConcurrentRequestLimit() { return -1; } - protected final CompletableFuture fetchVersionMetadata(Executor executor, String id, String url, String sha1, Path targetDir, String targetFileKind, Class metadataClass) throws IOException { + protected final CompletableFuture fetchVersionMetadata(Executor executor, String id, String url, String sha1, Path targetDir, String targetFileKind, Class metadataClass, boolean useEtag) throws IOException { URI uri = null; try { uri = new URI(url); @@ -226,7 +226,7 @@ protected final CompletableFuture fetchVersionMetadata(Executor executor, String fileNameWithoutExt = fileName.substring(0, fileName.lastIndexOf('.')); String fileExt = fileName.lastIndexOf(".") != -1 ? fileName.substring(fileName.lastIndexOf(".") + 1) : ""; Path filePath = sha1 != null ? targetDir.resolve(String.format("%s_%s.%s", fileNameWithoutExt, sha1, fileExt)) : targetDir.resolve(fileName); // allow multiple files with same hash to coexist (in case of reuploads with same meta name, referenced from different versions) - CompletableFuture status = FileSystemNetworkManager.fetchRemoteSerialFSAccess(executor, uri, new FileSystemNetworkManager.LocalFileInfo(filePath, sha1, sha1 != null ? Library.IA_SHA1 : null, targetFileKind, id), true, false, this.getConcurrentRequestLimit()); + CompletableFuture status = FileSystemNetworkManager.fetchRemoteSerialFSAccess(executor, uri, new FileSystemNetworkManager.LocalFileInfo(filePath, sha1, sha1 != null ? Library.IA_SHA1 : null, targetFileKind, id), true, false, this.getConcurrentRequestLimit(), useEtag); return status.thenApply($ -> { try { return this.loadVersionMetadata(filePath, metadataClass, fileName); @@ -237,7 +237,7 @@ protected final CompletableFuture fetchVersionMetadata(Executor executor, }); } - protected final CompletableFuture fetchVersionMetadataFilename(Executor executor, String filename, String id, String url, String sha1, Path targetDir, String targetFileKind, Class metadataClass) throws IOException { + protected final CompletableFuture fetchVersionMetadataFilename(Executor executor, String filename, String id, String url, String sha1, Path targetDir, String targetFileKind, Class metadataClass, boolean useEtag) throws IOException { URI uri = null; try { uri = new URI(url); @@ -245,7 +245,7 @@ protected final CompletableFuture fetchVersionMetadataFilename(Executor e throw new IOException(e); } Path filePath = targetDir.resolve(filename); - CompletableFuture status = FileSystemNetworkManager.fetchRemoteSerialFSAccess(executor, uri, new FileSystemNetworkManager.LocalFileInfo(filePath, sha1, sha1 != null ? Library.IA_SHA1 : null, targetFileKind, id), true, false, this.getConcurrentRequestLimit()); + CompletableFuture status = FileSystemNetworkManager.fetchRemoteSerialFSAccess(executor, uri, new FileSystemNetworkManager.LocalFileInfo(filePath, sha1, sha1 != null ? Library.IA_SHA1 : null, targetFileKind, id), true, false, this.getConcurrentRequestLimit(), useEtag); return status.thenApply($ -> { try { return this.loadVersionMetadata(filePath, metadataClass, filePath.getFileName().toString()); diff --git a/src/main/groovy/com/github/winplay02/gitcraft/manifest/historic/HistoricMojangLauncherMetadataProvider.java b/src/main/groovy/com/github/winplay02/gitcraft/manifest/historic/HistoricMojangLauncherMetadataProvider.java index a24700c..fc6969b 100644 --- a/src/main/groovy/com/github/winplay02/gitcraft/manifest/historic/HistoricMojangLauncherMetadataProvider.java +++ b/src/main/groovy/com/github/winplay02/gitcraft/manifest/historic/HistoricMojangLauncherMetadataProvider.java @@ -234,7 +234,7 @@ protected void loadVersions(Executor executor) throws IOException { @Override protected CompletableFuture loadVersionFromManifest(Executor executor, MojangLauncherManifest.VersionEntry manifestEntry, Path targetDir) throws IOException { - CompletableFuture futureInfo = this.fetchVersionMetadata(executor, manifestEntry.id(), manifestEntry.url(), manifestEntry.sha1(), targetDir, "version info", VersionInfo.class); + CompletableFuture futureInfo = this.fetchVersionMetadata(executor, manifestEntry.id(), manifestEntry.url(), manifestEntry.sha1(), targetDir, "version info", VersionInfo.class, false); return futureInfo.thenApply(info -> { info = info.withUpdatedId(manifestEntry.id()); String semanticVersion = this.mojangLauncherMetadataProvider.lookupSemanticVersion(executor, info); diff --git a/src/main/groovy/com/github/winplay02/gitcraft/manifest/skyrising/SkyrisingMetadataProvider.java b/src/main/groovy/com/github/winplay02/gitcraft/manifest/skyrising/SkyrisingMetadataProvider.java index 71744c2..436d614 100644 --- a/src/main/groovy/com/github/winplay02/gitcraft/manifest/skyrising/SkyrisingMetadataProvider.java +++ b/src/main/groovy/com/github/winplay02/gitcraft/manifest/skyrising/SkyrisingMetadataProvider.java @@ -65,7 +65,7 @@ public int getConcurrentRequestLimit() { public CompletableFuture fetchSpecificManifest(Executor executor, String id, VersionDetails.ManifestEntry manifestEntryPtr) { try { - return this.fetchVersionMetadataFilename(executor, String.format("%s_%s.json", id, manifestEntryPtr.hash()), id, manifestEntryPtr.url(), manifestEntryPtr.hash(), this.manifestMetadata.resolve("manifests"), "version manifest", VersionInfo.class); + return this.fetchVersionMetadataFilename(executor, String.format("%s_%s.json", id, manifestEntryPtr.hash()), id, manifestEntryPtr.url(), manifestEntryPtr.hash(), this.manifestMetadata.resolve("manifests"), "version manifest", VersionInfo.class, true); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -73,8 +73,8 @@ public CompletableFuture fetchSpecificManifest(Executor executor, S @Override protected CompletableFuture loadVersionFromManifest(Executor executor, SkyrisingManifest.VersionEntry manifestEntry, Path targetDir) throws IOException { - CompletableFuture infoFuture = this.fetchVersionMetadata(executor, manifestEntry.id(), manifestEntry.url(), null, targetDir.resolve("info"), "version info", VersionInfo.class); - CompletableFuture detailsFuture = this.fetchVersionMetadata(executor, manifestEntry.id(), manifestEntry.details(), null, targetDir.resolve("details"), "version details", VersionDetails.class); + CompletableFuture infoFuture = this.fetchVersionMetadata(executor, manifestEntry.id(), manifestEntry.url(), null, targetDir.resolve("info"), "version info", VersionInfo.class, true); + CompletableFuture detailsFuture = this.fetchVersionMetadata(executor, manifestEntry.id(), manifestEntry.details(), null, targetDir.resolve("details"), "version details", VersionDetails.class, true); return CompletableFuture.allOf(infoFuture, detailsFuture).thenApply($ -> { VersionInfo info = infoFuture.join(); VersionDetails details = detailsFuture.join(); diff --git a/src/main/groovy/com/github/winplay02/gitcraft/manifest/vanilla/MojangLauncherMetadataProvider.java b/src/main/groovy/com/github/winplay02/gitcraft/manifest/vanilla/MojangLauncherMetadataProvider.java index 2233a4a..e27e440 100644 --- a/src/main/groovy/com/github/winplay02/gitcraft/manifest/vanilla/MojangLauncherMetadataProvider.java +++ b/src/main/groovy/com/github/winplay02/gitcraft/manifest/vanilla/MojangLauncherMetadataProvider.java @@ -211,7 +211,7 @@ public String getInternalName() { @Override protected CompletableFuture loadVersionFromManifest(Executor executor, MojangLauncherManifest.VersionEntry manifestEntry, Path targetDir) throws IOException { - CompletableFuture futureInfo = this.fetchVersionMetadata(executor, manifestEntry.id(), manifestEntry.url(), manifestEntry.sha1(), targetDir, "version info", VersionInfo.class); + CompletableFuture futureInfo = this.fetchVersionMetadata(executor, manifestEntry.id(), manifestEntry.url(), manifestEntry.sha1(), targetDir, "version info", VersionInfo.class, false); return futureInfo.thenApply(info -> { String semanticVersion = this.lookupSemanticVersion(executor, info); return OrderedVersion.from(info, semanticVersion);