Download less each run (#73)

* Drastically reduce the amount of downloading Loom does

Uses ETags and last modify times to avoid downloading the version manifests, the game jars and assets

* Documentation is good

* Avoid string concatenation with debug off

src/main/java/net/fabricmc/loom/providers/MinecraftLibraryProvider.java

@@ -28,11 +28,11 @@ import com.google.gson.Gson;
 import net.fabricmc.loom.LoomGradleExtension;
 import net.fabricmc.loom.util.Checksum;
 import net.fabricmc.loom.util.Constants;
+import net.fabricmc.loom.util.DownloadUtil;
 import net.fabricmc.loom.util.MinecraftVersionInfo;
 import net.fabricmc.loom.util.assets.AssetIndex;
 import net.fabricmc.loom.util.assets.AssetObject;
 import net.fabricmc.loom.util.progress.ProgressLogger;
-import org.apache.commons.io.FileUtils;
 import org.gradle.api.Project;
 
 import java.io.File;
@@ -81,7 +81,7 @@ public class MinecraftLibraryProvider {
 		File assetsInfo = new File(assets, "indexes" + File.separator + assetIndex.getFabricId(minecraftProvider.minecraftVersion) + ".json");
 		if (!assetsInfo.exists() || !Checksum.equals(assetsInfo, assetIndex.sha1)) {
 			project.getLogger().lifecycle(":downloading asset index");
-			FileUtils.copyURLToFile(new URL(assetIndex.url), assetsInfo);
+			DownloadUtil.downloadIfChanged(new URL(assetIndex.url), assetsInfo, project.getLogger());
 		}
 
 		ProgressLogger progressLogger = ProgressLogger.getProgressFactory(project, getClass().getName());
@@ -101,7 +101,7 @@ public class MinecraftLibraryProvider {
 
 			if (!file.exists() || !Checksum.equals(file, sha1)) {
 				project.getLogger().debug(":downloading asset " + entry.getKey());
-				FileUtils.copyURLToFile(new URL(Constants.RESOURCES_BASE + sha1.substring(0, 2) + "/" + sha1), file);
+				DownloadUtil.downloadIfChanged(new URL(Constants.RESOURCES_BASE + sha1.substring(0, 2) + "/" + sha1), file, project.getLogger(), true);
 			}
 			String assetName = entry.getKey();
 			int end = assetName.lastIndexOf("/") + 1;

src/main/java/net/fabricmc/loom/providers/MinecraftProvider.java

@@ -24,13 +24,14 @@
 
 package net.fabricmc.loom.providers;
 
+import com.google.common.io.Files;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import net.fabricmc.loom.LoomGradleExtension;
 import net.fabricmc.loom.util.*;
-import org.apache.commons.io.FileUtils;
+
-import org.apache.commons.io.IOUtils;
 import org.gradle.api.Project;
+import org.gradle.api.logging.Logger;
 
 import java.io.File;
 import java.io.FileReader;
@@ -61,7 +62,7 @@ public class MinecraftProvider extends DependencyProvider {
 
 		initFiles(project);
 
-		downloadMcJson();
+		downloadMcJson(project);
 		FileReader reader = new FileReader(MINECRAFT_JSON);
 		versionInfo = gson.fromJson(reader, MinecraftVersionInfo.class);
 		reader.close();
@@ -69,7 +70,7 @@ public class MinecraftProvider extends DependencyProvider {
 		// Add Loom as an annotation processor
         addDependency(project.files(this.getClass().getProtectionDomain().getCodeSource().getLocation()), project, "compileOnly");
 
-		downloadJars();
+		downloadJars(project.getLogger());
 
 		libraryProvider = new MinecraftLibraryProvider();
 		libraryProvider.provide(this, project);
@@ -85,26 +86,34 @@ public class MinecraftProvider extends DependencyProvider {
 
 	}
 
-	private void downloadMcJson() throws IOException {
+	private void downloadMcJson(Project project) throws IOException {
-		String versionManifest = IOUtils.toString(new URL("https://launchermeta.mojang.com/mc/game/version_manifest.json"), StandardCharsets.UTF_8);
+		LoomGradleExtension extension = project.getExtensions().getByType(LoomGradleExtension.class);
+		File manifests = new File(extension.getUserCache(), "version_manifest.json");
+
+		project.getLogger().debug("Downloading version manifests");
+		DownloadUtil.downloadIfChanged(new URL("https://launchermeta.mojang.com/mc/game/version_manifest.json"), manifests, project.getLogger());
+		String versionManifest = Files.asCharSource(manifests, StandardCharsets.UTF_8).read();
 		ManifestVersion mcManifest = new GsonBuilder().create().fromJson(versionManifest, ManifestVersion.class);
 
 		Optional<ManifestVersion.Versions> optionalVersion = mcManifest.versions.stream().filter(versions -> versions.id.equalsIgnoreCase(minecraftVersion)).findFirst();
 		if (optionalVersion.isPresent()) {
-			FileUtils.copyURLToFile(new URL(optionalVersion.get().url), MINECRAFT_JSON);
+			project.getLogger().debug("Downloading Minecraft {} manifest", minecraftVersion);
+			DownloadUtil.downloadIfChanged(new URL(optionalVersion.get().url), MINECRAFT_JSON, project.getLogger());
 		} else {
 			throw new RuntimeException("Failed to find minecraft version: " + minecraftVersion);
 		}
 
 	}
 
-	private void downloadJars() throws IOException {
+	private void downloadJars(Logger logger) throws IOException {
 		if (!MINECRAFT_CLIENT_JAR.exists() || !Checksum.equals(MINECRAFT_CLIENT_JAR, versionInfo.downloads.get("client").sha1)) {
-			FileUtils.copyURLToFile(new URL(versionInfo.downloads.get("client").url), MINECRAFT_CLIENT_JAR);
+			logger.debug("Downloading Minecraft {} client jar", minecraftVersion);
+			DownloadUtil.downloadIfChanged(new URL(versionInfo.downloads.get("client").url), MINECRAFT_CLIENT_JAR, logger);
 		}
 
 		if (!MINECRAFT_SERVER_JAR.exists() || !Checksum.equals(MINECRAFT_SERVER_JAR, versionInfo.downloads.get("server").sha1)) {
-			FileUtils.copyURLToFile(new URL(versionInfo.downloads.get("server").url), MINECRAFT_SERVER_JAR);
+			logger.debug("Downloading Minecraft {} server jar", minecraftVersion);
+			DownloadUtil.downloadIfChanged(new URL(versionInfo.downloads.get("server").url), MINECRAFT_SERVER_JAR, logger);
 		}
 	}
 

src/main/java/net/fabricmc/loom/util/DownloadUtil.java

@@ -0,0 +1,183 @@
+/*
+ * This file is part of fabric-loom, licensed under the MIT License (MIT).
+ *
+ * Copyright (c) 2019 Chocohead
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package net.fabricmc.loom.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.commons.io.FileUtils;
+
+import org.gradle.api.Project;
+import org.gradle.api.logging.Logger;
+
+import com.google.common.io.Files;
+
+public class DownloadUtil {
+	/**
+	 * Download from the given {@link URL} to the given {@link File} so long as there are differences between them
+	 *
+	 * @param from The URL of the file to be downloaded
+	 * @param to The destination to be saved to, and compared against if it exists
+	 * @param logger The logger to print everything to, typically from {@link Project#getLogger()}
+	 *
+	 * @throws IOException If an exception occurs during the process
+	 */
+	public static void downloadIfChanged(URL from, File to, Logger logger) throws IOException {
+		downloadIfChanged(from, to, logger, false);
+	}
+
+	/**
+	 * Download from the given {@link URL} to the given {@link File} so long as there are differences between them
+	 *
+	 * @param from The URL of the file to be downloaded
+	 * @param to The destination to be saved to, and compared against if it exists
+	 * @param logger The logger to print information to, typically from {@link Project#getLogger()}
+	 * @param quiet Whether to only print warnings (when <code>true</code>) or everything
+	 *
+	 * @throws IOException If an exception occurs during the process
+	 */
+	public static void downloadIfChanged(URL from, File to, Logger logger, boolean quiet) throws IOException {
+		HttpURLConnection connection = (HttpURLConnection) from.openConnection();
+
+		//If the output already exists we'll use it's last modified time
+		if (to.exists()) connection.setIfModifiedSince(to.lastModified());
+
+		//Try use the ETag if there's one for the file we're downloading
+		String etag = loadETag(to, logger);
+		if (etag != null) connection.setRequestProperty("If-None-Match", etag);
+
+		//We want to download gzip compressed stuff
+		connection.setRequestProperty("Accept-Encoding", "gzip");
+
+		//We shouldn't need to set a user agent, but it's here just in case
+		//connection.setRequestProperty("User-Agent", null);
+
+		//Try make the connection, it will hang here if the connection is bad
+		connection.connect();
+
+		int code = connection.getResponseCode();
+		if ((code < 200 || code > 299) && code != HttpURLConnection.HTTP_NOT_MODIFIED) {
+			//Didn't get what we expected
+			throw new IOException(connection.getResponseMessage());
+		}
+
+		long modifyTime = connection.getHeaderFieldDate("Last-Modified", -1);
+		if (code == HttpURLConnection.HTTP_NOT_MODIFIED || modifyTime > 0 && to.exists() && to.lastModified() >= modifyTime) {
+			if (!quiet) logger.info("'{}' Not Modified, skipping.", to);
+			return; //What we've got is already fine
+		}
+
+		long contentLength = connection.getContentLengthLong();
+		if (!quiet && contentLength >= 0) logger.info("'{}' Changed, downloading {}", to, toNiceSize(contentLength));
+
+		try {//Try download to the output
+			FileUtils.copyInputStreamToFile(connection.getInputStream(), to);
+		} catch (IOException e) {
+			to.delete(); //Probably isn't good if it fails to copy/save
+			throw e;
+		}
+
+		//Set the modify time to match the server's (if we know it)
+		if (modifyTime > 0) to.setLastModified(modifyTime);
+
+		//Save the ETag (if we know it)
+		String eTag = connection.getHeaderField("ETag");
+		if (eTag != null) {
+			//Log if we get a weak ETag and we're not on quiet
+			if (!quiet && eTag.startsWith("W/")) logger.warn("Weak ETag found.");
+
+			saveETag(to, eTag, logger);
+		}
+	}
+
+	/**
+	 * Creates a new file in the same directory as the given file with <code>.etag</code> on the end of the name
+	 *
+	 * @param file The file to produce the ETag for
+	 *
+	 * @return The (uncreated) ETag file for the given file
+	 */
+	private static File getETagFile(File file) {
+		return new File(file.getAbsoluteFile().getParentFile(), file.getName() + ".etag");
+	}
+
+	/**
+	 * Attempt to load an ETag for the given file, if it exists
+	 *
+	 * @param to The file to load an ETag for
+	 * @param logger The logger to print errors to if it goes wrong
+	 *
+	 * @return The ETag for the given file, or <code>null</code> if it doesn't exist
+	 */
+	private static String loadETag(File to, Logger logger) {
+		File eTagFile = getETagFile(to);
+		if (!eTagFile.exists()) return null;
+
+		try {
+			return Files.asCharSource(eTagFile, StandardCharsets.UTF_8).read();
+		} catch (IOException e) {
+			logger.warn("Error reading ETag file '{}'.", eTagFile);
+			return null;
+		}
+	}
+
+	/**
+	 * Saves the given ETag for the given file, replacing it if it already exists
+	 *
+	 * @param to The file to save the ETag for
+	 * @param eTag The ETag to be saved
+	 * @param logger The logger to print errors to if it goes wrong
+	 */
+	private static void saveETag(File to, String eTag, Logger logger) {
+		File eTagFile = getETagFile(to);
+		try {
+			if (!eTagFile.exists()) eTagFile.createNewFile();
+			Files.asCharSink(eTagFile, StandardCharsets.UTF_8).write(eTag);
+		} catch (IOException e) {
+			logger.warn("Error saving ETag file '{}'.", eTagFile, e);
+		}
+	}
+
+	/**
+	 * Format the given number of bytes as a more human readable string
+	 *
+	 * @param bytes The number of bytes
+	 *
+	 * @return The given number of bytes formatted to kilobytes, megabytes or gigabytes if appropriate
+	 */
+	private static String toNiceSize(long bytes) {
+		if (bytes < 1024) {
+			return bytes + " B";
+		} else if (bytes < 1024 * 1024) {
+			return bytes / 1024 + " KB";
+		} else if (bytes < 1024 * 1024 * 1024) {
+			return String.format("%.2f MB", bytes / (1024.0 * 1024.0));
+		} else {
+			return String.format("%.2f GB", bytes / (1024.0 * 1024.0 * 1024.0));
+		}
+	}
+}