Use ETag header to check validity of manifests which do not have hashes

build.gradle

@@ -74,6 +74,7 @@ dependencies {
 	implementation "net.fabricmc:access-widener:${access_widener_version}"
 	implementation "net.fabricmc:mapping-io:${mappingio_version}"
 	implementation "net.fabricmc:fabric-loom:${loom_version}"
+	libImplementation "net.fabricmc:fabric-loom:${loom_version}"
 	implementation "net.fabricmc.unpick:unpick:${unpick_version}"
 	implementation "net.fabricmc.unpick:unpick-format-utils:${unpick_version}"
 

src/lib/java/com/github/winplay02/gitcraft/util/FileSystemNetworkManager.java

@@ -3,6 +3,7 @@
 import com.github.winplay02.gitcraft.Library;
 import com.github.winplay02.gitcraft.integrity.IntegrityAlgorithm;
 import com.github.winplay02.gitcraft.pipeline.StepStatus;
+import net.fabricmc.loom.util.AttributeHelper;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -14,11 +15,13 @@
 import java.net.http.HttpResponse;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
 import java.nio.file.StandardOpenOption;
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
@@ -57,15 +60,15 @@ private static LockGuard acquireDownloadJobsReadLock() {
 
 	protected static final Map<Path, NetworkProgressInfo> completedJobs = new ConcurrentHashMap<>();
 
-	public static CompletableFuture<StepStatus> fetchRemoteSerialFSAccess(Executor executor, URI url, LocalFileInfo localFileInfo, boolean retry, boolean tolerateHashUnavailable, int concurrentLimit) {
+	public static CompletableFuture<StepStatus> fetchRemoteSerialFSAccess(Executor executor, URI url, LocalFileInfo localFileInfo, boolean retry, boolean tolerateHashUnavailable, int concurrentLimit, boolean useEtag) {
 		if (completedJobs.containsKey(localFileInfo.targetFile()) &&
-			Objects.equals(completedJobs.get(localFileInfo.targetFile()).integrityChecksum, localFileInfo.checksum()) &&
-			Objects.equals(completedJobs.get(localFileInfo.targetFile()).integrityAlgorithm, localFileInfo.integrityAlgorithm())) {
+			Objects.equals(completedJobs.get(localFileInfo.targetFile()).integrityChecksum(), localFileInfo.checksum()) &&
+			Objects.equals(completedJobs.get(localFileInfo.targetFile()).integrityAlgorithm(), localFileInfo.integrityAlgorithm())) {
 			return CompletableFuture.completedFuture(StepStatus.UP_TO_DATE);
 		}
 		if (completedJobs.containsKey(localFileInfo.targetFile()) &&
-			!Objects.equals(completedJobs.get(localFileInfo.targetFile()).integrityChecksum, localFileInfo.checksum()) &&
-			Objects.equals(completedJobs.get(localFileInfo.targetFile()).integrityAlgorithm, localFileInfo.integrityAlgorithm())) {
+			!Objects.equals(completedJobs.get(localFileInfo.targetFile()).integrityChecksum(), localFileInfo.checksum()) &&
+			Objects.equals(completedJobs.get(localFileInfo.targetFile()).integrityAlgorithm(), localFileInfo.integrityAlgorithm())) {
 			MiscHelper.panic("Cannot fulfill download to %s, there are multiple requests with different outcomes to the same file", localFileInfo.targetFile());
 		}
 		try (LockGuard $ = acquireDownloadJobsWriteLock()) {
@@ -83,7 +86,10 @@ public static CompletableFuture<StepStatus> fetchRemoteSerialFSAccess(Executor e
 					try {
 						MiscHelper.println("Fetching %s %s from: %s", localFileInfo.outputFileKind(), localFileInfo.outputFileId(), url);
 						try {
-							FileSystemNetworkManager.fetchFileAsync(url, localFileInfo.targetFile(), concurrentLimit).get();
+							HttpResponse<Path> response = FileSystemNetworkManager.fetchFileAsync(url, localFileInfo.targetFile(), concurrentLimit, useEtag).get();
+							if (useEtag && response.statusCode() == 304) {
+								MiscHelper.println("Skipped downloading %s %s as it is already up-to-date", localFileInfo.outputFileKind(), localFileInfo.outputFileId());
+							}
 							if (!retry) {
 								break;
 							}
@@ -118,9 +124,22 @@ public static CompletableFuture<StepStatus> fetchRemoteSerialFSAccess(Executor e
 	protected static final HttpClient httpClient = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NORMAL).build();
 
 	protected static final Map<String, Semaphore> connectionLimiter = new ConcurrentHashMap<>();
-
-	protected static CompletableFuture<HttpResponse<Path>> fetchFileAsync(URI uri, Path targetFile, int concurrentLimit) {
-		HttpRequest request = HttpRequest.newBuilder(uri).GET().build();
+
+	private static final String ETAG_ATTRIBUTE = "ETag";
+
+	protected static CompletableFuture<HttpResponse<Path>> fetchFileAsync(URI uri, Path targetFile, int concurrentLimit, boolean useEtag) {
+		HttpRequest.Builder builder = HttpRequest.newBuilder(uri).GET();
+		if (useEtag) {
+			try {
+				Optional<String> etag = AttributeHelper.readAttribute(targetFile, ETAG_ATTRIBUTE);
+				if (etag.isPresent()) {
+					builder.header("If-None-Match", etag.orElseThrow());
+				}
+			} catch (IOException e) {
+				MiscHelper.panicBecause(e, "Cannot read etag for %s", targetFile);
+			}
+		}
+		HttpRequest request = builder.build();
 		final Semaphore semaphore = connectionLimiter.computeIfAbsent(uri.getHost().toLowerCase(Locale.ROOT), $ ->
 				new Semaphore(concurrentLimit > 0 ?
 						Math.min(concurrentLimit, Library.CONF_GLOBAL.maxConcurrentHttpRequestsPerOrigin())
@@ -133,12 +152,47 @@ protected static CompletableFuture<HttpResponse<Path>> fetchFileAsync(URI uri, P
 			}
 		}
 		semaphore.acquireUninterruptibly();
-		return httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofFile(targetFile, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE, StandardOpenOption.WRITE)).thenApply(response -> {
-			if (response.statusCode() == 404) {
-				MiscHelper.throwUnchecked(new FileNotFoundException(uri.toString()));
-			}
-			return response;
-		}).whenComplete(($, $$) -> semaphore.release());
+		if (useEtag) {
+			Path tmpDownloadFile = targetFile.resolveSibling(targetFile.getFileName() + ".tmp"); // we have to download to temporary location because empty 304 Not Modified response erases file contents
+			return httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofFile(tmpDownloadFile, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE, StandardOpenOption.WRITE)).thenApply(response -> {
+				if (response.statusCode() >= 200 && response.statusCode() < 300) {
+					try {
+						Files.move(tmpDownloadFile, targetFile, StandardCopyOption.REPLACE_EXISTING);
+
+						Optional<String> received_etag = response.headers().firstValue("etag");
+						if (received_etag.isPresent()) {
+							AttributeHelper.writeAttribute(targetFile, ETAG_ATTRIBUTE, received_etag.orElseThrow());
+						}
+					} catch (IOException e) {
+						try {
+							Files.deleteIfExists(tmpDownloadFile);
+						} catch (IOException e1) {
+							MiscHelper.throwUnchecked(e1);
+						}
+
+						MiscHelper.throwUnchecked(e);
+					}
+				}
+
+				try {
+					Files.deleteIfExists(tmpDownloadFile);
+				} catch (IOException e) {
+					MiscHelper.throwUnchecked(e);
+				}
+
+				if (response.statusCode() == 404) {
+					MiscHelper.throwUnchecked(new FileNotFoundException(uri.toString()));
+				}
+				return response;
+			}).whenComplete(($, $$) -> semaphore.release());
+		} else {
+			return httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofFile(targetFile, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE, StandardOpenOption.WRITE)).thenApply(response -> {
+				if (response.statusCode() == 404) {
+					MiscHelper.throwUnchecked(new FileNotFoundException(uri.toString()));
+				}
+				return response;
+			}).whenComplete(($, $$) -> semaphore.release());
+		}
 	}
 
 	public static String fetchAllFromURLSync(URL url) throws IOException, URISyntaxException, InterruptedException {

src/lib/java/com/github/winplay02/gitcraft/util/RemoteHelper.java

@@ -40,7 +40,7 @@ public static StepStatus downloadToFileWithChecksumIfNotExists(Executor executor
 		}
 		while (true) {
 			try {
-				return FileSystemNetworkManager.fetchRemoteSerialFSAccess(executor, uri, localFileInfo, retry, false, -1).get();
+				return FileSystemNetworkManager.fetchRemoteSerialFSAccess(executor, uri, localFileInfo, retry, false, -1, false).get();
 			} catch (InterruptedException e) {
 				MiscHelper.println("Interrupted while waiting for download of %s to complete", url);
 				continue;

src/main/groovy/com/github/winplay02/gitcraft/manifest/BaseMetadataProvider.java

@@ -215,7 +215,7 @@ public int getConcurrentRequestLimit() {
 		return -1;
 	}
 
-	protected final <T> CompletableFuture<T> fetchVersionMetadata(Executor executor, String id, String url, String sha1, Path targetDir, String targetFileKind, Class<T> metadataClass) throws IOException {
+	protected final <T> CompletableFuture<T> fetchVersionMetadata(Executor executor, String id, String url, String sha1, Path targetDir, String targetFileKind, Class<T> metadataClass, boolean useEtag) throws IOException {
 		URI uri = null;
 		try {
 			uri = new URI(url);
@@ -226,7 +226,7 @@ protected final <T> CompletableFuture<T> fetchVersionMetadata(Executor executor,
 		String fileNameWithoutExt = fileName.substring(0, fileName.lastIndexOf('.'));
 		String fileExt = fileName.lastIndexOf(".") != -1 ? fileName.substring(fileName.lastIndexOf(".") + 1) : "";
 		Path filePath = sha1 != null ? targetDir.resolve(String.format("%s_%s.%s", fileNameWithoutExt, sha1, fileExt)) : targetDir.resolve(fileName); // allow multiple files with same hash to coexist (in case of reuploads with same meta name, referenced from different versions)
-		CompletableFuture<StepStatus> status = FileSystemNetworkManager.fetchRemoteSerialFSAccess(executor, uri, new FileSystemNetworkManager.LocalFileInfo(filePath, sha1, sha1 != null ? Library.IA_SHA1 : null, targetFileKind, id), true, false, this.getConcurrentRequestLimit());
+		CompletableFuture<StepStatus> status = FileSystemNetworkManager.fetchRemoteSerialFSAccess(executor, uri, new FileSystemNetworkManager.LocalFileInfo(filePath, sha1, sha1 != null ? Library.IA_SHA1 : null, targetFileKind, id), true, false, this.getConcurrentRequestLimit(), useEtag);
 		return status.thenApply($ -> {
 			try {
 				return this.loadVersionMetadata(filePath, metadataClass, fileName);
@@ -237,15 +237,15 @@ protected final <T> CompletableFuture<T> fetchVersionMetadata(Executor executor,
 		});
 	}
 
-	protected final <T> CompletableFuture<T> fetchVersionMetadataFilename(Executor executor, String filename, String id, String url, String sha1, Path targetDir, String targetFileKind, Class<T> metadataClass) throws IOException {
+	protected final <T> CompletableFuture<T> fetchVersionMetadataFilename(Executor executor, String filename, String id, String url, String sha1, Path targetDir, String targetFileKind, Class<T> metadataClass, boolean useEtag) throws IOException {
 		URI uri = null;
 		try {
 			uri = new URI(url);
 		} catch (URISyntaxException e) {
 			throw new IOException(e);
 		}
 		Path filePath = targetDir.resolve(filename);
-		CompletableFuture<StepStatus> status = FileSystemNetworkManager.fetchRemoteSerialFSAccess(executor, uri, new FileSystemNetworkManager.LocalFileInfo(filePath, sha1, sha1 != null ? Library.IA_SHA1 : null, targetFileKind, id), true, false, this.getConcurrentRequestLimit());
+		CompletableFuture<StepStatus> status = FileSystemNetworkManager.fetchRemoteSerialFSAccess(executor, uri, new FileSystemNetworkManager.LocalFileInfo(filePath, sha1, sha1 != null ? Library.IA_SHA1 : null, targetFileKind, id), true, false, this.getConcurrentRequestLimit(), useEtag);
 		return status.thenApply($ -> {
 			try {
 				return this.loadVersionMetadata(filePath, metadataClass, filePath.getFileName().toString());

src/main/groovy/com/github/winplay02/gitcraft/manifest/historic/HistoricMojangLauncherMetadataProvider.java

@@ -234,7 +234,7 @@ protected void loadVersions(Executor executor) throws IOException {
 
 	@Override
 	protected CompletableFuture<OrderedVersion> loadVersionFromManifest(Executor executor, MojangLauncherManifest.VersionEntry manifestEntry, Path targetDir) throws IOException {
-		CompletableFuture<VersionInfo> futureInfo = this.fetchVersionMetadata(executor, manifestEntry.id(), manifestEntry.url(), manifestEntry.sha1(), targetDir, "version info", VersionInfo.class);
+		CompletableFuture<VersionInfo> futureInfo = this.fetchVersionMetadata(executor, manifestEntry.id(), manifestEntry.url(), manifestEntry.sha1(), targetDir, "version info", VersionInfo.class, false);
 		return futureInfo.thenApply(info -> {
 			info = info.withUpdatedId(manifestEntry.id());
 			String semanticVersion = this.mojangLauncherMetadataProvider.lookupSemanticVersion(executor, info);

src/main/groovy/com/github/winplay02/gitcraft/manifest/skyrising/SkyrisingMetadataProvider.java

@@ -65,16 +65,16 @@ public int getConcurrentRequestLimit() {
 
 	public CompletableFuture<VersionInfo> fetchSpecificManifest(Executor executor, String id, VersionDetails.ManifestEntry manifestEntryPtr) {
 		try {
-			return this.fetchVersionMetadataFilename(executor, String.format("%s_%s.json", id, manifestEntryPtr.hash()), id, manifestEntryPtr.url(), manifestEntryPtr.hash(), this.manifestMetadata.resolve("manifests"), "version manifest", VersionInfo.class);
+			return this.fetchVersionMetadataFilename(executor, String.format("%s_%s.json", id, manifestEntryPtr.hash()), id, manifestEntryPtr.url(), manifestEntryPtr.hash(), this.manifestMetadata.resolve("manifests"), "version manifest", VersionInfo.class, true);
 		} catch (IOException e) {
 			throw new UncheckedIOException(e);
 		}
 	}
 
 	@Override
 	protected CompletableFuture<OrderedVersion> loadVersionFromManifest(Executor executor, SkyrisingManifest.VersionEntry manifestEntry, Path targetDir) throws IOException {
-		CompletableFuture<VersionInfo> infoFuture = this.fetchVersionMetadata(executor, manifestEntry.id(), manifestEntry.url(), null, targetDir.resolve("info"), "version info", VersionInfo.class);
-		CompletableFuture<VersionDetails> detailsFuture = this.fetchVersionMetadata(executor, manifestEntry.id(), manifestEntry.details(), null, targetDir.resolve("details"), "version details", VersionDetails.class);
+		CompletableFuture<VersionInfo> infoFuture = this.fetchVersionMetadata(executor, manifestEntry.id(), manifestEntry.url(), null, targetDir.resolve("info"), "version info", VersionInfo.class, true);
+		CompletableFuture<VersionDetails> detailsFuture = this.fetchVersionMetadata(executor, manifestEntry.id(), manifestEntry.details(), null, targetDir.resolve("details"), "version details", VersionDetails.class, true);
 		return CompletableFuture.allOf(infoFuture, detailsFuture).thenApply($ -> {
 			VersionInfo info = infoFuture.join();
 			VersionDetails details = detailsFuture.join();

src/main/groovy/com/github/winplay02/gitcraft/manifest/vanilla/MojangLauncherMetadataProvider.java

@@ -211,7 +211,7 @@ public String getInternalName() {
 
 	@Override
 	protected CompletableFuture<OrderedVersion> loadVersionFromManifest(Executor executor, MojangLauncherManifest.VersionEntry manifestEntry, Path targetDir) throws IOException {
-		CompletableFuture<VersionInfo> futureInfo = this.fetchVersionMetadata(executor, manifestEntry.id(), manifestEntry.url(), manifestEntry.sha1(), targetDir, "version info", VersionInfo.class);
+		CompletableFuture<VersionInfo> futureInfo = this.fetchVersionMetadata(executor, manifestEntry.id(), manifestEntry.url(), manifestEntry.sha1(), targetDir, "version info", VersionInfo.class, false);
 		return futureInfo.thenApply(info -> {
 			String semanticVersion = this.lookupSemanticVersion(executor, info);
 			return OrderedVersion.from(info, semanticVersion);