hibiscus-client/fabric-loom
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Download less each run (#73)

Browse source 
* Drastically reduce the amount of downloading Loom does

Uses ETags and last modify times to avoid downloading the version manifests, the game jars and assets

* Documentation is good

* Avoid string concatenation with debug off
This commit is contained in:
Chocohead 2019-04-19 19:15:48 +01:00 committed by Modmuss50
parent e72ccc104c
commit a55ebd4e31
3 changed files with 205 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/main/java/net/fabricmc/loom/providers/MinecraftLibraryProvider.java
View file

@ -28,11 +28,11 @@ import com.google.gson.Gson;
import net.fabricmc.loom.LoomGradleExtension;
import net.fabricmc.loom.util.Checksum;
import net.fabricmc.loom.util.Constants;
import net.fabricmc.loom.util.DownloadUtil;
import net.fabricmc.loom.util.MinecraftVersionInfo;
import net.fabricmc.loom.util.assets.AssetIndex;
import net.fabricmc.loom.util.assets.AssetObject;
import net.fabricmc.loom.util.progress.ProgressLogger;
import org.apache.commons.io.FileUtils;
import org.gradle.api.Project;
import java.io.File;
@ -81,7 +81,7 @@ public class MinecraftLibraryProvider {
File assetsInfo = new File(assets, "indexes" + File.separator + assetIndex.getFabricId(minecraftProvider.minecraftVersion) + ".json");
if (!assetsInfo.exists() || !Checksum.equals(assetsInfo, assetIndex.sha1)) {
project.getLogger().lifecycle(":downloading asset index");
FileUtils.copyURLToFile(new URL(assetIndex.url), assetsInfo);
DownloadUtil.downloadIfChanged(new URL(assetIndex.url), assetsInfo, project.getLogger());
}
ProgressLogger progressLogger = ProgressLogger.getProgressFactory(project, getClass().getName());
@ -101,7 +101,7 @@ public class MinecraftLibraryProvider {
if (!file.exists() || !Checksum.equals(file, sha1)) {
project.getLogger().debug(":downloading asset " + entry.getKey());
FileUtils.copyURLToFile(new URL(Constants.RESOURCES_BASE + sha1.substring(0, 2) + "/" + sha1), file);
DownloadUtil.downloadIfChanged(new URL(Constants.RESOURCES_BASE + sha1.substring(0, 2) + "/" + sha1), file, project.getLogger(), true);
}
String assetName = entry.getKey();
int end = assetName.lastIndexOf("/") + 1;

29
src/main/java/net/fabricmc/loom/providers/MinecraftProvider.java
View file

@ -24,13 +24,14 @@
package net.fabricmc.loom.providers;
import com.google.common.io.Files;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import net.fabricmc.loom.LoomGradleExtension;
import net.fabricmc.loom.util.*;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.gradle.api.Project;
import org.gradle.api.logging.Logger;
import java.io.File;
import java.io.FileReader;
@ -61,7 +62,7 @@ public class MinecraftProvider extends DependencyProvider {
initFiles(project);
downloadMcJson();
downloadMcJson(project);
FileReader reader = new FileReader(MINECRAFT_JSON);
versionInfo = gson.fromJson(reader, MinecraftVersionInfo.class);
reader.close();
@ -69,7 +70,7 @@ public class MinecraftProvider extends DependencyProvider {
// Add Loom as an annotation processor
addDependency(project.files(this.getClass().getProtectionDomain().getCodeSource().getLocation()), project, "compileOnly");
downloadJars();
downloadJars(project.getLogger());
libraryProvider = new MinecraftLibraryProvider();
libraryProvider.provide(this, project);
@ -85,26 +86,34 @@ public class MinecraftProvider extends DependencyProvider {
}
private void downloadMcJson() throws IOException {
String versionManifest = IOUtils.toString(new URL("https://launchermeta.mojang.com/mc/game/version_manifest.json"), StandardCharsets.UTF_8);
private void downloadMcJson(Project project) throws IOException {
LoomGradleExtension extension = project.getExtensions().getByType(LoomGradleExtension.class);
File manifests = new File(extension.getUserCache(), "version_manifest.json");
project.getLogger().debug("Downloading version manifests");
DownloadUtil.downloadIfChanged(new URL("https://launchermeta.mojang.com/mc/game/version_manifest.json"), manifests, project.getLogger());
String versionManifest = Files.asCharSource(manifests, StandardCharsets.UTF_8).read();
ManifestVersion mcManifest = new GsonBuilder().create().fromJson(versionManifest, ManifestVersion.class);
Optional<ManifestVersion.Versions> optionalVersion = mcManifest.versions.stream().filter(versions -> versions.id.equalsIgnoreCase(minecraftVersion)).findFirst();
if (optionalVersion.isPresent()) {
FileUtils.copyURLToFile(new URL(optionalVersion.get().url), MINECRAFT_JSON);
project.getLogger().debug("Downloading Minecraft {} manifest", minecraftVersion);
DownloadUtil.downloadIfChanged(new URL(optionalVersion.get().url), MINECRAFT_JSON, project.getLogger());
} else {
throw new RuntimeException("Failed to find minecraft version: " + minecraftVersion);
}
}
private void downloadJars() throws IOException {
private void downloadJars(Logger logger) throws IOException {
if (!MINECRAFT_CLIENT_JAR.exists() || !Checksum.equals(MINECRAFT_CLIENT_JAR, versionInfo.downloads.get("client").sha1)) {
FileUtils.copyURLToFile(new URL(versionInfo.downloads.get("client").url), MINECRAFT_CLIENT_JAR);
logger.debug("Downloading Minecraft {} client jar", minecraftVersion);
DownloadUtil.downloadIfChanged(new URL(versionInfo.downloads.get("client").url), MINECRAFT_CLIENT_JAR, logger);
}
if (!MINECRAFT_SERVER_JAR.exists() || !Checksum.equals(MINECRAFT_SERVER_JAR, versionInfo.downloads.get("server").sha1)) {
FileUtils.copyURLToFile(new URL(versionInfo.downloads.get("server").url), MINECRAFT_SERVER_JAR);
logger.debug("Downloading Minecraft {} server jar", minecraftVersion);
DownloadUtil.downloadIfChanged(new URL(versionInfo.downloads.get("server").url), MINECRAFT_SERVER_JAR, logger);
}
}

183
src/main/java/net/fabricmc/loom/util/DownloadUtil.java Normal file
View file

@ -0,0 +1,183 @@
/*
* This file is part of fabric-loom, licensed under the MIT License (MIT).
*
* Copyright (c) 2019 Chocohead
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package net.fabricmc.loom.util;
import java.io.File;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import org.apache.commons.io.FileUtils;
import org.gradle.api.Project;
import org.gradle.api.logging.Logger;
import com.google.common.io.Files;
public class DownloadUtil {
/**
* Download from the given {@link URL} to the given {@link File} so long as there are differences between them
*
* @param from The URL of the file to be downloaded
* @param to The destination to be saved to, and compared against if it exists
* @param logger The logger to print everything to, typically from {@link Project#getLogger()}
*
* @throws IOException If an exception occurs during the process
*/
public static void downloadIfChanged(URL from, File to, Logger logger) throws IOException {
downloadIfChanged(from, to, logger, false);
}
/**
* Download from the given {@link URL} to the given {@link File} so long as there are differences between them
*
* @param from The URL of the file to be downloaded
* @param to The destination to be saved to, and compared against if it exists
* @param logger The logger to print information to, typically from {@link Project#getLogger()}
* @param quiet Whether to only print warnings (when <code>true</code>) or everything
*
* @throws IOException If an exception occurs during the process
*/
public static void downloadIfChanged(URL from, File to, Logger logger, boolean quiet) throws IOException {
HttpURLConnection connection = (HttpURLConnection) from.openConnection();
//If the output already exists we'll use it's last modified time
if (to.exists()) connection.setIfModifiedSince(to.lastModified());
//Try use the ETag if there's one for the file we're downloading
String etag = loadETag(to, logger);
if (etag != null) connection.setRequestProperty("If-None-Match", etag);
//We want to download gzip compressed stuff
connection.setRequestProperty("Accept-Encoding", "gzip");
//We shouldn't need to set a user agent, but it's here just in case
//connection.setRequestProperty("User-Agent", null);
//Try make the connection, it will hang here if the connection is bad
connection.connect();
int code = connection.getResponseCode();
if ((code < 200 || code > 299) && code != HttpURLConnection.HTTP_NOT_MODIFIED) {
//Didn't get what we expected
throw new IOException(connection.getResponseMessage());
}
long modifyTime = connection.getHeaderFieldDate("Last-Modified", -1);
if (code == HttpURLConnection.HTTP_NOT_MODIFIED || modifyTime > 0 && to.exists() && to.lastModified() >= modifyTime) {
if (!quiet) logger.info("'{}' Not Modified, skipping.", to);
return; //What we've got is already fine
}
long contentLength = connection.getContentLengthLong();
if (!quiet && contentLength >= 0) logger.info("'{}' Changed, downloading {}", to, toNiceSize(contentLength));
try {//Try download to the output
FileUtils.copyInputStreamToFile(connection.getInputStream(), to);
} catch (IOException e) {
to.delete(); //Probably isn't good if it fails to copy/save
throw e;
}
//Set the modify time to match the server's (if we know it)
if (modifyTime > 0) to.setLastModified(modifyTime);
//Save the ETag (if we know it)
String eTag = connection.getHeaderField("ETag");
if (eTag != null) {
//Log if we get a weak ETag and we're not on quiet
if (!quiet && eTag.startsWith("W/")) logger.warn("Weak ETag found.");
saveETag(to, eTag, logger);
}
}
/**
* Creates a new file in the same directory as the given file with <code>.etag</code> on the end of the name
*
* @param file The file to produce the ETag for
*
* @return The (uncreated) ETag file for the given file
*/
private static File getETagFile(File file) {
return new File(file.getAbsoluteFile().getParentFile(), file.getName() + ".etag");
}
/**
* Attempt to load an ETag for the given file, if it exists
*
* @param to The file to load an ETag for
* @param logger The logger to print errors to if it goes wrong
*
* @return The ETag for the given file, or <code>null</code> if it doesn't exist
*/
private static String loadETag(File to, Logger logger) {
File eTagFile = getETagFile(to);
if (!eTagFile.exists()) return null;
try {
return Files.asCharSource(eTagFile, StandardCharsets.UTF_8).read();
} catch (IOException e) {
logger.warn("Error reading ETag file '{}'.", eTagFile);
return null;
}
}
/**
* Saves the given ETag for the given file, replacing it if it already exists
*
* @param to The file to save the ETag for
* @param eTag The ETag to be saved
* @param logger The logger to print errors to if it goes wrong
*/
private static void saveETag(File to, String eTag, Logger logger) {
File eTagFile = getETagFile(to);
try {
if (!eTagFile.exists()) eTagFile.createNewFile();
Files.asCharSink(eTagFile, StandardCharsets.UTF_8).write(eTag);
} catch (IOException e) {
logger.warn("Error saving ETag file '{}'.", eTagFile, e);
}
}
/**
* Format the given number of bytes as a more human readable string
*
* @param bytes The number of bytes
*
* @return The given number of bytes formatted to kilobytes, megabytes or gigabytes if appropriate
*/
private static String toNiceSize(long bytes) {
if (bytes < 1024) {
return bytes + " B";
} else if (bytes < 1024 * 1024) {
return bytes / 1024 + " KB";
} else if (bytes < 1024 * 1024 * 1024) {
return String.format("%.2f MB", bytes / (1024.0 * 1024.0));
} else {
return String.format("%.2f GB", bytes / (1024.0 * 1024.0 * 1024.0));
}
}
}