From 792a64e2efecb850e4752a298a4e3d6f50f55a64 Mon Sep 17 00:00:00 2001 From: modmuss50 Date: Sun, 27 Dec 2020 16:25:30 +0000 Subject: [PATCH] Optimise loom configuration, saves 1.2 seconds in my testing. #319 --- .../fabricmc/loom/LoomGradleExtension.java | 6 +- .../net/fabricmc/loom/LoomGradlePlugin.java | 3 + .../loom/build/MixinRefmapHelper.java | 12 +- .../loom/build/ModCompileRemapper.java | 19 ++- .../net/fabricmc/loom/build/NestedJars.java | 11 +- .../configuration/CompileConfiguration.java | 17 +-- .../configuration/DependencyProvider.java | 12 +- .../configuration/LoomDependencyManager.java | 2 +- .../loom/configuration/mods/ModProcessor.java | 11 +- .../dependency/ModDependencyInfo.java | 8 +- .../providers/MinecraftProvider.java | 71 ++++++--- .../providers/mappings/MappingsProvider.java | 10 +- .../fabricmc/loom/task/AbstractRunTask.java | 6 +- .../java/net/fabricmc/loom/util/Checksum.java | 13 +- .../net/fabricmc/loom/util/DownloadUtil.java | 2 +- .../loom/util/HashedDownloadUtil.java | 136 ++++++++++++++++++ 16 files changed, 252 insertions(+), 87 deletions(-) create mode 100644 src/main/java/net/fabricmc/loom/util/HashedDownloadUtil.java diff --git a/src/main/java/net/fabricmc/loom/LoomGradleExtension.java b/src/main/java/net/fabricmc/loom/LoomGradleExtension.java index dfbcf8d..22ee092 100644 --- a/src/main/java/net/fabricmc/loom/LoomGradleExtension.java +++ b/src/main/java/net/fabricmc/loom/LoomGradleExtension.java @@ -235,10 +235,8 @@ public class LoomGradleExtension { } public File getNativesDirectory() { - Object customNativesDir = project.getProperties().get("fabric.loom.natives.dir"); - - if (customNativesDir != null) { - return new File((String) customNativesDir); + if (project.hasProperty("fabric.loom.natives.dir")) { + return new File((String) project.property("fabric.loom.natives.dir")); } File natives = new File(getUserCache(), "natives/" + getMinecraftProvider().getMinecraftVersion()); diff --git a/src/main/java/net/fabricmc/loom/LoomGradlePlugin.java b/src/main/java/net/fabricmc/loom/LoomGradlePlugin.java index e9486d4..cac0b32 100644 --- a/src/main/java/net/fabricmc/loom/LoomGradlePlugin.java +++ b/src/main/java/net/fabricmc/loom/LoomGradlePlugin.java @@ -25,6 +25,8 @@ package net.fabricmc.loom; import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import org.gradle.api.Plugin; import org.gradle.api.Project; @@ -38,6 +40,7 @@ import net.fabricmc.loom.task.LoomTasks; public class LoomGradlePlugin implements Plugin { public static boolean refreshDeps; + public static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); @Override public void apply(Project project) { diff --git a/src/main/java/net/fabricmc/loom/build/MixinRefmapHelper.java b/src/main/java/net/fabricmc/loom/build/MixinRefmapHelper.java index dd90e2f..dace689 100644 --- a/src/main/java/net/fabricmc/loom/build/MixinRefmapHelper.java +++ b/src/main/java/net/fabricmc/loom/build/MixinRefmapHelper.java @@ -32,16 +32,14 @@ import java.util.HashSet; import java.util.Set; import java.util.zip.ZipEntry; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; import com.google.gson.JsonObject; import org.zeroturnaround.zip.ZipUtil; import org.zeroturnaround.zip.transform.StringZipEntryTransformer; import org.zeroturnaround.zip.transform.ZipEntryTransformerEntry; -public final class MixinRefmapHelper { - private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); +import net.fabricmc.loom.LoomGradlePlugin; +public final class MixinRefmapHelper { private MixinRefmapHelper() { } public static boolean addRefmapName(String filename, String mixinVersion, Path outputPath) { @@ -52,7 +50,7 @@ public final class MixinRefmapHelper { return ZipUtil.transformEntries(output, mixinFilenames.stream().map((f) -> new ZipEntryTransformerEntry(f, new StringZipEntryTransformer("UTF-8") { @Override protected String transform(ZipEntry zipEntry, String input) throws IOException { - JsonObject json = GSON.fromJson(input, JsonObject.class); + JsonObject json = LoomGradlePlugin.GSON.fromJson(input, JsonObject.class); if (!json.has("refmap")) { json.addProperty("refmap", filename); @@ -62,7 +60,7 @@ public final class MixinRefmapHelper { json.addProperty("minVersion", mixinVersion); } - return GSON.toJson(json); + return LoomGradlePlugin.GSON.toJson(json); } })).toArray(ZipEntryTransformerEntry[]::new)); } else { @@ -78,7 +76,7 @@ public final class MixinRefmapHelper { if (!entry.isDirectory() && entry.getName().endsWith(".json") && !entry.getName().contains("/") && !entry.getName().contains("\\")) { // JSON file in root directory try (InputStreamReader inputStreamReader = new InputStreamReader(stream)) { - JsonObject json = GSON.fromJson(inputStreamReader, JsonObject.class); + JsonObject json = LoomGradlePlugin.GSON.fromJson(inputStreamReader, JsonObject.class); if (json != null) { boolean hasMixins = json.has("mixins") && json.get("mixins").isJsonArray(); diff --git a/src/main/java/net/fabricmc/loom/build/ModCompileRemapper.java b/src/main/java/net/fabricmc/loom/build/ModCompileRemapper.java index eb9d87e..5fc7771 100644 --- a/src/main/java/net/fabricmc/loom/build/ModCompileRemapper.java +++ b/src/main/java/net/fabricmc/loom/build/ModCompileRemapper.java @@ -52,6 +52,7 @@ import net.fabricmc.loom.configuration.processors.dependency.RemapData; import net.fabricmc.loom.util.Constants; import net.fabricmc.loom.util.SourceRemapper; +@SuppressWarnings("UnstableApiUsage") public class ModCompileRemapper { public static void remapDependencies(Project project, String mappingsSuffix, LoomGradleExtension extension, SourceRemapper sourceRemapper) { Logger logger = project.getLogger(); @@ -82,9 +83,7 @@ public class ModCompileRemapper { continue; } - File sources = findSources(dependencies, artifact); - - ModDependencyInfo info = new ModDependencyInfo(group, name, version, classifierSuffix, artifact.getFile(), sources, remappedConfig, remapData); + ModDependencyInfo info = new ModDependencyInfo(group, name, version, classifierSuffix, artifact.getFile(), remappedConfig, remapData); if (refreshDeps) { info.forceRemap(); @@ -95,8 +94,14 @@ public class ModCompileRemapper { String remappedLog = group + ":" + name + ":" + version + classifierSuffix + " (" + mappingsSuffix + ")"; project.getLogger().info(":providing " + remappedLog); - if (sources != null) { - scheduleSourcesRemapping(project, sourceRemapper, info.sourcesFile, info.getRemappedNotation(), info.getRemappedOutput(), modStore); + File remappedSources = new File(info.getRemappedOutput().getAbsolutePath().replace(".jar", "-sources.jar")); + + if (!remappedSources.exists() || refreshDeps) { + File sources = findSources(dependencies, artifact); + + if (sources != null) { + scheduleSourcesRemapping(project, sourceRemapper, sources, info.getRemappedNotation(), info.getRemappedOutput(), modStore); + } } } @@ -145,8 +150,8 @@ public class ModCompileRemapper { } public static File findSources(DependencyHandler dependencies, ResolvedArtifact artifact) { - @SuppressWarnings("unchecked") ArtifactResolutionQuery query = dependencies.createArtifactResolutionQuery()// - .forComponents(artifact.getId().getComponentIdentifier())// + @SuppressWarnings("unchecked") ArtifactResolutionQuery query = dependencies.createArtifactResolutionQuery() + .forComponents(artifact.getId().getComponentIdentifier()) .withArtifacts(JvmLibrary.class, SourcesArtifact.class); for (ComponentArtifactsResult result : query.execute().getResolvedComponents()) { diff --git a/src/main/java/net/fabricmc/loom/build/NestedJars.java b/src/main/java/net/fabricmc/loom/build/NestedJars.java index c088af4..172efca 100644 --- a/src/main/java/net/fabricmc/loom/build/NestedJars.java +++ b/src/main/java/net/fabricmc/loom/build/NestedJars.java @@ -36,8 +36,6 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.zip.ZipEntry; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; import com.google.gson.JsonArray; import com.google.gson.JsonObject; import org.apache.commons.io.FileUtils; @@ -58,12 +56,11 @@ import org.zeroturnaround.zip.transform.StringZipEntryTransformer; import org.zeroturnaround.zip.transform.ZipEntryTransformerEntry; import net.fabricmc.loom.LoomGradleExtension; +import net.fabricmc.loom.LoomGradlePlugin; import net.fabricmc.loom.task.RemapJarTask; import net.fabricmc.loom.util.Constants; public class NestedJars { - private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); - public static boolean addNestedJars(Project project, Path modJarPath) { List containedJars = getContainedJars(project); @@ -78,7 +75,7 @@ public class NestedJars { return ZipUtil.transformEntries(modJar, single(new ZipEntryTransformerEntry("fabric.mod.json", new StringZipEntryTransformer() { @Override protected String transform(ZipEntry zipEntry, String input) { - JsonObject json = GSON.fromJson(input, JsonObject.class); + JsonObject json = LoomGradlePlugin.GSON.fromJson(input, JsonObject.class); JsonArray nestedJars = json.getAsJsonArray("jars"); if (nestedJars == null || !json.has("jars")) { @@ -93,7 +90,7 @@ public class NestedJars { json.add("jars", nestedJars); - return GSON.toJson(json); + return LoomGradlePlugin.GSON.toJson(json); } }))); } @@ -229,7 +226,7 @@ public class NestedJars { custom.addProperty("fabric-loom:generated", true); jsonObject.add("custom", custom); - return GSON.toJson(jsonObject); + return LoomGradlePlugin.GSON.toJson(jsonObject); } private static ZipEntryTransformerEntry[] single(ZipEntryTransformerEntry element) { diff --git a/src/main/java/net/fabricmc/loom/configuration/CompileConfiguration.java b/src/main/java/net/fabricmc/loom/configuration/CompileConfiguration.java index b5f7ee2..8b264a0 100644 --- a/src/main/java/net/fabricmc/loom/configuration/CompileConfiguration.java +++ b/src/main/java/net/fabricmc/loom/configuration/CompileConfiguration.java @@ -25,8 +25,6 @@ package net.fabricmc.loom.configuration; import java.io.IOException; -import java.util.Map; -import java.util.Set; import org.gradle.api.Project; import org.gradle.api.Task; @@ -198,18 +196,11 @@ public final class CompileConfiguration { remapJarTask.dependsOn(jarTask); project1.getTasks().getByName("build").dependsOn(remapJarTask); - Map> taskMap = project.getAllTasks(true); - - for (Map.Entry> entry : taskMap.entrySet()) { - Set taskSet = entry.getValue(); - - for (Task task : taskSet) { - if (task instanceof RemapJarTask && ((RemapJarTask) task).getAddNestedDependencies().getOrElse(false)) { - //Run all the sub project remap jars tasks before the root projects jar, this is to allow us to include projects - NestedJars.getRequiredTasks(project1).forEach(task::dependsOn); - } + project.getTasks().withType(RemapJarTask.class).forEach(task -> { + if (task.getAddNestedDependencies().getOrElse(false)) { + NestedJars.getRequiredTasks(project1).forEach(task::dependsOn); } - } + }); SourceRemapper remapper = null; Task parentTask = project1.getTasks().getByName("build"); diff --git a/src/main/java/net/fabricmc/loom/configuration/DependencyProvider.java b/src/main/java/net/fabricmc/loom/configuration/DependencyProvider.java index 4b1b800..1795d3b 100644 --- a/src/main/java/net/fabricmc/loom/configuration/DependencyProvider.java +++ b/src/main/java/net/fabricmc/loom/configuration/DependencyProvider.java @@ -100,6 +100,8 @@ public abstract class DependencyProvider { final Dependency dependency; final Configuration sourceConfiguration; + private String resolvedVersion = null; + public static DependencyInfo create(Project project, Dependency dependency, Configuration sourceConfiguration) { if (dependency instanceof SelfResolvingDependency) { return new FileDependencyInfo(project, (SelfResolvingDependency) dependency, sourceConfiguration); @@ -119,13 +121,19 @@ public abstract class DependencyProvider { } public String getResolvedVersion() { + if (resolvedVersion != null) { + return resolvedVersion; + } + for (ResolvedDependency rd : sourceConfiguration.getResolvedConfiguration().getFirstLevelModuleDependencies()) { if (rd.getModuleGroup().equals(dependency.getGroup()) && rd.getModuleName().equals(dependency.getName())) { - return rd.getModuleVersion(); + resolvedVersion = rd.getModuleVersion(); + return resolvedVersion; } } - return dependency.getVersion(); + resolvedVersion = dependency.getVersion(); + return resolvedVersion; } public Configuration getSourceConfiguration() { diff --git a/src/main/java/net/fabricmc/loom/configuration/LoomDependencyManager.java b/src/main/java/net/fabricmc/loom/configuration/LoomDependencyManager.java index 2bd1fb5..5397794 100644 --- a/src/main/java/net/fabricmc/loom/configuration/LoomDependencyManager.java +++ b/src/main/java/net/fabricmc/loom/configuration/LoomDependencyManager.java @@ -87,7 +87,7 @@ public class LoomDependencyManager { MappingsProvider mappingsProvider = null; - project.getLogger().lifecycle(":setting up loom dependencies"); + project.getLogger().info(":setting up loom dependencies"); LoomGradleExtension extension = project.getExtensions().getByType(LoomGradleExtension.class); Map providerListMap = new HashMap<>(); List targetProviders = new ArrayList<>(); diff --git a/src/main/java/net/fabricmc/loom/configuration/mods/ModProcessor.java b/src/main/java/net/fabricmc/loom/configuration/mods/ModProcessor.java index 3837004..367a49a 100644 --- a/src/main/java/net/fabricmc/loom/configuration/mods/ModProcessor.java +++ b/src/main/java/net/fabricmc/loom/configuration/mods/ModProcessor.java @@ -41,8 +41,6 @@ import java.util.jar.JarFile; import java.util.stream.Collectors; import java.util.zip.ZipEntry; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; import com.google.gson.JsonObject; import org.apache.commons.io.IOUtils; import org.gradle.api.Project; @@ -56,6 +54,7 @@ import net.fabricmc.accesswidener.AccessWidenerReader; import net.fabricmc.accesswidener.AccessWidenerRemapper; import net.fabricmc.accesswidener.AccessWidenerWriter; import net.fabricmc.loom.LoomGradleExtension; +import net.fabricmc.loom.LoomGradlePlugin; import net.fabricmc.loom.configuration.RemappedConfigurationEntry; import net.fabricmc.loom.configuration.processors.dependency.ModDependencyInfo; import net.fabricmc.loom.configuration.providers.mappings.MappingsProvider; @@ -67,8 +66,6 @@ import net.fabricmc.tinyremapper.OutputConsumerPath; import net.fabricmc.tinyremapper.TinyRemapper; public class ModProcessor { - public static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); - public static void processMods(Project project, List processList) throws IOException { if (processList.stream().noneMatch(ModDependencyInfo::requiresRemapping)) { return; @@ -104,9 +101,9 @@ public class ModProcessor { ZipUtil.transformEntries(file, new ZipEntryTransformerEntry[] {(new ZipEntryTransformerEntry("fabric.mod.json", new StringZipEntryTransformer() { @Override protected String transform(ZipEntry zipEntry, String input) { - JsonObject json = GSON.fromJson(input, JsonObject.class); + JsonObject json = LoomGradlePlugin.GSON.fromJson(input, JsonObject.class); json.remove("jars"); - return GSON.toJson(json); + return LoomGradlePlugin.GSON.toJson(json); } }))}); } @@ -236,7 +233,7 @@ public class ModProcessor { } } - return GSON.fromJson(jsonStr, JsonObject.class); + return LoomGradlePlugin.GSON.fromJson(jsonStr, JsonObject.class); } catch (IOException e) { e.printStackTrace(); } diff --git a/src/main/java/net/fabricmc/loom/configuration/processors/dependency/ModDependencyInfo.java b/src/main/java/net/fabricmc/loom/configuration/processors/dependency/ModDependencyInfo.java index 95872aa..1f6682f 100644 --- a/src/main/java/net/fabricmc/loom/configuration/processors/dependency/ModDependencyInfo.java +++ b/src/main/java/net/fabricmc/loom/configuration/processors/dependency/ModDependencyInfo.java @@ -37,7 +37,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.gradle.api.artifacts.Configuration; -import net.fabricmc.loom.configuration.mods.ModProcessor; +import net.fabricmc.loom.LoomGradlePlugin; public class ModDependencyInfo { private final String group; @@ -45,20 +45,18 @@ public class ModDependencyInfo { public final String version; public final String classifier; public final File inputFile; - public final File sourcesFile; public final Configuration targetConfig; public final RemapData remapData; private boolean forceRemap = false; - public ModDependencyInfo(String group, String name, String version, String classifier, File inputFile, File sourcesFile, Configuration targetConfig, RemapData remapData) { + public ModDependencyInfo(String group, String name, String version, String classifier, File inputFile, Configuration targetConfig, RemapData remapData) { this.group = group; this.name = name; this.version = version; this.classifier = classifier; this.inputFile = inputFile; - this.sourcesFile = sourcesFile; this.targetConfig = targetConfig; this.remapData = remapData; } @@ -149,7 +147,7 @@ public class ModDependencyInfo { } try (InputStream inputStream = jarFile.getInputStream(modJsonEntry)) { - JsonObject json = ModProcessor.GSON.fromJson(new InputStreamReader(inputStream), JsonObject.class); + JsonObject json = LoomGradlePlugin.GSON.fromJson(new InputStreamReader(inputStream), JsonObject.class); if (!json.has("accessWidener")) { return null; diff --git a/src/main/java/net/fabricmc/loom/configuration/providers/MinecraftProvider.java b/src/main/java/net/fabricmc/loom/configuration/providers/MinecraftProvider.java index d6df8ad..7a9737f 100644 --- a/src/main/java/net/fabricmc/loom/configuration/providers/MinecraftProvider.java +++ b/src/main/java/net/fabricmc/loom/configuration/providers/MinecraftProvider.java @@ -31,22 +31,24 @@ import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.Optional; import java.util.function.Consumer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.zip.ZipError; import com.google.common.io.Files; -import com.google.gson.Gson; import com.google.gson.GsonBuilder; import org.gradle.api.GradleException; import org.gradle.api.Project; import org.gradle.api.logging.Logger; +import net.fabricmc.loom.LoomGradlePlugin; import net.fabricmc.loom.configuration.DependencyProvider; import net.fabricmc.loom.configuration.providers.minecraft.ManifestVersion; import net.fabricmc.loom.configuration.providers.minecraft.MinecraftLibraryProvider; import net.fabricmc.loom.configuration.providers.minecraft.MinecraftVersionInfo; import net.fabricmc.loom.util.Constants; import net.fabricmc.loom.util.DownloadUtil; -import net.fabricmc.loom.util.StaticPathWatcher; +import net.fabricmc.loom.util.HashedDownloadUtil; import net.fabricmc.stitch.merge.JarMerger; public class MinecraftProvider extends DependencyProvider { @@ -59,8 +61,7 @@ public class MinecraftProvider extends DependencyProvider { private File minecraftClientJar; private File minecraftServerJar; private File minecraftMergedJar; - - Gson gson = new Gson(); + private File versionManifestJson; public MinecraftProvider(Project project) { super(project); @@ -77,7 +78,7 @@ public class MinecraftProvider extends DependencyProvider { downloadMcJson(offline); try (FileReader reader = new FileReader(minecraftJson)) { - versionInfo = gson.fromJson(reader, MinecraftVersionInfo.class); + versionInfo = LoomGradlePlugin.GSON.fromJson(reader, MinecraftVersionInfo.class); } // Add Loom as an annotation processor @@ -117,29 +118,30 @@ public class MinecraftProvider extends DependencyProvider { minecraftClientJar = new File(getExtension().getUserCache(), "minecraft-" + minecraftVersion + "-client.jar"); minecraftServerJar = new File(getExtension().getUserCache(), "minecraft-" + minecraftVersion + "-server.jar"); minecraftMergedJar = new File(getExtension().getUserCache(), "minecraft-" + minecraftVersion + "-merged.jar"); + versionManifestJson = new File(getExtension().getUserCache(), "version_manifest.json"); } private void downloadMcJson(boolean offline) throws IOException { - File manifests = new File(getExtension().getUserCache(), "version_manifest.json"); - - if (getExtension().isShareCaches() && !getExtension().isRootProject() && manifests.exists() && !isRefreshDeps()) { + if (getExtension().isShareCaches() && !getExtension().isRootProject() && versionManifestJson.exists() && !isRefreshDeps()) { return; } - if (offline) { - if (manifests.exists()) { + if (!offline && !isRefreshDeps() && hasRecentValidManifest()) { + // We have a recent valid manifest file, so do nothing + } else if (offline) { + if (versionManifestJson.exists()) { // If there is the manifests already we'll presume that's good enough getProject().getLogger().debug("Found version manifests, presuming up-to-date"); } else { // If we don't have the manifests then there's nothing more we can do - throw new GradleException("Version manifests not found at " + manifests.getAbsolutePath()); + throw new GradleException("Version manifests not found at " + versionManifestJson.getAbsolutePath()); } } else { getProject().getLogger().debug("Downloading version manifests"); - DownloadUtil.downloadIfChanged(new URL(Constants.VERSION_MANIFESTS), manifests, getProject().getLogger()); + DownloadUtil.downloadIfChanged(new URL(Constants.VERSION_MANIFESTS), versionManifestJson, getProject().getLogger()); } - String versionManifest = Files.asCharSource(manifests, StandardCharsets.UTF_8).read(); + String versionManifest = Files.asCharSource(versionManifestJson, StandardCharsets.UTF_8).read(); ManifestVersion mcManifest = new GsonBuilder().create().fromJson(versionManifest, ManifestVersion.class); Optional optionalVersion = Optional.empty(); @@ -166,9 +168,18 @@ public class MinecraftProvider extends DependencyProvider { throw new GradleException("Minecraft " + minecraftVersion + " manifest not found at " + minecraftJson.getAbsolutePath()); } } else { - if (StaticPathWatcher.INSTANCE.hasFileChanged(minecraftJson.toPath()) || isRefreshDeps()) { - getProject().getLogger().debug("Downloading Minecraft {} manifest", minecraftVersion); - DownloadUtil.downloadIfChanged(new URL(optionalVersion.get().url), minecraftJson, getProject().getLogger()); + getProject().getLogger().debug("Downloading Minecraft {} manifest", minecraftVersion); + + String url = optionalVersion.get().url; + // Find the sha1 of the json from the url, return true if it matches the local json + Pattern sha1Pattern = Pattern.compile("\\b[0-9a-f]{5,40}\\b"); + Matcher matcher = sha1Pattern.matcher(url); + + if (matcher.find()) { + HashedDownloadUtil.downloadIfInvalid(new URL(url), minecraftJson, matcher.group(), getProject().getLogger(), true); + } else { + // Use the etag if no hash found from url + DownloadUtil.downloadIfChanged(new URL(url), minecraftJson, getProject().getLogger()); } } } else { @@ -176,13 +187,37 @@ public class MinecraftProvider extends DependencyProvider { } } + private boolean hasRecentValidManifest() throws IOException { + if (getExtension().customManifest != null) { + return false; + } + + if (!versionManifestJson.exists() || !minecraftJson.exists()) { + return false; + } + + if (versionManifestJson.lastModified() > System.currentTimeMillis() - 24 * 3_600_000) { + // Version manifest hasn't been modified in 24 hours, time to get a new one. + return false; + } + + ManifestVersion manifest = new GsonBuilder().create().fromJson(Files.asCharSource(versionManifestJson, StandardCharsets.UTF_8).read(), ManifestVersion.class); + Optional version = manifest.versions.stream().filter(versions -> versions.id.equalsIgnoreCase(minecraftVersion)).findFirst(); + + // fail if the expected mc version was not found, will download the file again. + return version.isPresent(); + } + private void downloadJars(Logger logger) throws IOException { if (getExtension().isShareCaches() && !getExtension().isRootProject() && minecraftClientJar.exists() && minecraftServerJar.exists() && !isRefreshDeps()) { return; } - DownloadUtil.downloadIfChanged(new URL(versionInfo.downloads.get("client").url), minecraftClientJar, logger); - DownloadUtil.downloadIfChanged(new URL(versionInfo.downloads.get("server").url), minecraftServerJar, logger); + MinecraftVersionInfo.Downloads client = versionInfo.downloads.get("client"); + MinecraftVersionInfo.Downloads server = versionInfo.downloads.get("server"); + + HashedDownloadUtil.downloadIfInvalid(new URL(client.url), minecraftClientJar, client.sha1, logger, false); + HashedDownloadUtil.downloadIfInvalid(new URL(server.url), minecraftServerJar, server.sha1, logger, false); } private void mergeJars(Logger logger) throws IOException { diff --git a/src/main/java/net/fabricmc/loom/configuration/providers/mappings/MappingsProvider.java b/src/main/java/net/fabricmc/loom/configuration/providers/mappings/MappingsProvider.java index ba2d827..810a34f 100644 --- a/src/main/java/net/fabricmc/loom/configuration/providers/mappings/MappingsProvider.java +++ b/src/main/java/net/fabricmc/loom/configuration/providers/mappings/MappingsProvider.java @@ -97,7 +97,7 @@ public class MappingsProvider extends DependencyProvider { public void provide(DependencyInfo dependency, Consumer postPopulationScheduler) throws Exception { MinecraftProvider minecraftProvider = getDependencyManager().getProvider(MinecraftProvider.class); - getProject().getLogger().lifecycle(":setting up mappings (" + dependency.getDependency().getName() + " " + dependency.getResolvedVersion() + ")"); + getProject().getLogger().info(":setting up mappings (" + dependency.getDependency().getName() + " " + dependency.getResolvedVersion() + ")"); String version = dependency.getResolvedVersion(); File mappingsJar = dependency.resolveFile().orElseThrow(() -> new RuntimeException("Could not find yarn mappings: " + dependency)); @@ -105,6 +105,8 @@ public class MappingsProvider extends DependencyProvider { this.mappingsName = StringUtils.removeSuffix(dependency.getDependency().getGroup() + "." + dependency.getDependency().getName(), "-unmerged"); this.minecraftVersion = minecraftProvider.getMinecraftVersion(); + boolean isV2; + // Only do this for official yarn, there isn't really a way we can get the mc version for all mappings if (dependency.getDependency().getGroup() != null && dependency.getDependency().getGroup().equals("net.fabricmc") && dependency.getDependency().getName().equals("yarn") && dependency.getDependency().getVersion() != null) { String yarnVersion = dependency.getDependency().getVersion(); @@ -114,9 +116,13 @@ public class MappingsProvider extends DependencyProvider { if (!yarnMinecraftVersion.equalsIgnoreCase(minecraftVersion)) { throw new RuntimeException(String.format("Minecraft Version (%s) does not match yarn's minecraft version (%s)", minecraftVersion, yarnMinecraftVersion)); } + + // We can save reading the zip file + header by checking the file name + isV2 = mappingsJar.getName().endsWith("-v2.jar"); + } else { + isV2 = doesJarContainV2Mappings(mappingsJar.toPath()); } - boolean isV2 = doesJarContainV2Mappings(mappingsJar.toPath()); this.mappingsVersion = version + (isV2 ? "-v2" : ""); initFiles(); diff --git a/src/main/java/net/fabricmc/loom/task/AbstractRunTask.java b/src/main/java/net/fabricmc/loom/task/AbstractRunTask.java index 0f2f94a..a1bee96 100644 --- a/src/main/java/net/fabricmc/loom/task/AbstractRunTask.java +++ b/src/main/java/net/fabricmc/loom/task/AbstractRunTask.java @@ -45,9 +45,6 @@ public abstract class AbstractRunTask extends JavaExec { super(); setGroup("fabric"); this.configProvider = config; - - classpath(getProject().getConfigurations().getByName("runtimeClasspath")); - classpath(this.getProject().getExtensions().getByType(LoomGradleExtension.class).getUnmappedModCollection()); } @Override @@ -56,6 +53,9 @@ public abstract class AbstractRunTask extends JavaExec { config = configProvider.apply(getProject()); } + classpath(getProject().getConfigurations().getByName("runtimeClasspath")); + classpath(this.getProject().getExtensions().getByType(LoomGradleExtension.class).getUnmappedModCollection()); + List argsSplit = new ArrayList<>(); String[] args = config.programArgs.split(" "); int partPos = -1; diff --git a/src/main/java/net/fabricmc/loom/util/Checksum.java b/src/main/java/net/fabricmc/loom/util/Checksum.java index c83623e..1b38a05 100644 --- a/src/main/java/net/fabricmc/loom/util/Checksum.java +++ b/src/main/java/net/fabricmc/loom/util/Checksum.java @@ -37,21 +37,14 @@ public class Checksum { private static final Logger log = Logging.getLogger(Checksum.class); public static boolean equals(File file, String checksum) { - if (file == null) { + if (file == null || !file.exists()) { return false; } try { - //noinspection deprecation HashCode hash = Files.asByteSource(file).hash(Hashing.sha1()); - StringBuilder builder = new StringBuilder(); - - for (Byte hashBytes : hash.asBytes()) { - builder.append(Integer.toString((hashBytes & 0xFF) + 0x100, 16).substring(1)); - } - - log.debug("Checksum check: '" + builder.toString() + "' == '" + checksum + "'?"); - return builder.toString().equals(checksum); + log.debug("Checksum check: '" + hash.toString() + "' == '" + checksum + "'?"); + return hash.toString().equals(checksum); } catch (IOException e) { e.printStackTrace(); } diff --git a/src/main/java/net/fabricmc/loom/util/DownloadUtil.java b/src/main/java/net/fabricmc/loom/util/DownloadUtil.java index 506adae..9bb2b7e 100644 --- a/src/main/java/net/fabricmc/loom/util/DownloadUtil.java +++ b/src/main/java/net/fabricmc/loom/util/DownloadUtil.java @@ -192,7 +192,7 @@ public class DownloadUtil { * @param bytes The number of bytes * @return The given number of bytes formatted to kilobytes, megabytes or gigabytes if appropriate */ - private static String toNiceSize(long bytes) { + public static String toNiceSize(long bytes) { if (bytes < 1024) { return bytes + " B"; } else if (bytes < 1024 * 1024) { diff --git a/src/main/java/net/fabricmc/loom/util/HashedDownloadUtil.java b/src/main/java/net/fabricmc/loom/util/HashedDownloadUtil.java new file mode 100644 index 0000000..da294b8 --- /dev/null +++ b/src/main/java/net/fabricmc/loom/util/HashedDownloadUtil.java @@ -0,0 +1,136 @@ +/* + * This file is part of fabric-loom, licensed under the MIT License (MIT). + * + * Copyright (c) 2016, 2017, 2018 FabricMC + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package net.fabricmc.loom.util; + +import java.io.File; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; + +import javax.annotation.Nullable; + +import com.google.common.hash.Hashing; +import com.google.common.io.Files; +import org.apache.commons.io.FileUtils; +import org.gradle.api.logging.Logger; + +import net.fabricmc.loom.LoomGradlePlugin; + +public class HashedDownloadUtil { + public static void downloadIfInvalid(URL from, File to, String expectedHash, Logger logger, boolean quiet) throws IOException { + if (LoomGradlePlugin.refreshDeps) { + delete(to); + } + + if (to.exists()) { + String sha1 = getSha1(to, logger); + + if (expectedHash.equals(sha1)) { + // The hash in the sha1 file matches + return; + } + } + + HttpURLConnection connection = (HttpURLConnection) from.openConnection(); + connection.setRequestProperty("Accept-Encoding", "gzip"); + connection.connect(); + + int code = connection.getResponseCode(); + + if ((code < 200 || code > 299) && code != HttpURLConnection.HTTP_NOT_MODIFIED) { + //Didn't get what we expected + throw new IOException(connection.getResponseMessage() + " for " + from); + } + + long contentLength = connection.getContentLengthLong(); + + if (!quiet && contentLength >= 0) { + logger.info("'{}' Changed, downloading {}", to, DownloadUtil.toNiceSize(contentLength)); + } + + try { // Try download to the output + FileUtils.copyInputStreamToFile(connection.getInputStream(), to); + } catch (IOException e) { + delete(to); // Probably isn't good if it fails to copy/save + throw e; + } + + if (!Checksum.equals(to, expectedHash)) { + String actualHash = Files.asByteSource(to).hash(Hashing.sha1()).toString(); + delete(to); + + throw new IOException(String.format("Downloaded file from %s to %s and got unexpected hash of %s expected %s", from, to, actualHash, expectedHash)); + } + + saveSha1(to, expectedHash, logger); + } + + private static File getSha1File(File file) { + return new File(file.getAbsoluteFile().getParentFile(), file.getName() + ".sha1"); + } + + @Nullable + private static String getSha1(File to, Logger logger) { + File sha1File = getSha1File(to); + + if (!sha1File.exists()) { + return null; + } + + try { + return Files.asCharSource(sha1File, StandardCharsets.UTF_8).read(); + } catch (IOException e) { + logger.warn("Error reading sha1 file '{}'.", sha1File); + return null; + } + } + + private static void saveSha1(File to, String sha1, Logger logger) { + File sha1File = getSha1File(to); + + try { + if (!sha1File.exists()) { + sha1File.createNewFile(); + } + + Files.asCharSink(sha1File, StandardCharsets.UTF_8).write(sha1); + } catch (IOException e) { + logger.warn("Error saving sha1 file '{}'.", sha1File, e); + } + } + + public static void delete(File file) { + if (file.exists()) { + file.delete(); + } + + File sha1File = getSha1File(file); + + if (sha1File.exists()) { + sha1File.delete(); + } + } +}