From 3eff7d0fdb95c6ac53405abcf9873f6edfb44770 Mon Sep 17 00:00:00 2001 From: modmuss50 Date: Wed, 13 May 2020 19:44:45 +0100 Subject: [PATCH] Improve setup speed. (#208) * Improve access widener remapper, now uses tiny remapper * First pass on using the new tiny remapper * Optimise source remapping --- build.gradle | 2 +- .../fabricmc/loom/LoomGradleExtension.java | 1 - .../dependency/ModDependencyInfo.java | 104 +++++++++ .../loom/processors/dependency/RemapData.java | 37 ++++ .../providers/MinecraftMappedProvider.java | 50 ++++- .../net/fabricmc/loom/task/RemapJarTask.java | 13 +- .../loom/util/LoomDependencyManager.java | 6 +- .../net/fabricmc/loom/util/MapJarsTiny.java | 77 ------- .../loom/util/ModCompileRemapper.java | 90 ++++---- .../net/fabricmc/loom/util/ModProcessor.java | 208 ++++++------------ .../fabricmc/loom/util/SourceRemapper.java | 109 ++++++--- .../AccessWidenerJarProcessor.java | 14 +- .../accesswidener/AccessWidenerRemapper.java | 71 ++---- 13 files changed, 420 insertions(+), 362 deletions(-) create mode 100644 src/main/java/net/fabricmc/loom/processors/dependency/ModDependencyInfo.java create mode 100644 src/main/java/net/fabricmc/loom/processors/dependency/RemapData.java delete mode 100644 src/main/java/net/fabricmc/loom/util/MapJarsTiny.java diff --git a/build.gradle b/build.gradle index 655a998..9f5f64a 100644 --- a/build.gradle +++ b/build.gradle @@ -47,7 +47,7 @@ dependencies { } // tinyfile management - implementation ('net.fabricmc:tiny-remapper:0.2.2.66') { + implementation ('net.fabricmc:tiny-remapper:0.3.0.70') { transitive = false } diff --git a/src/main/java/net/fabricmc/loom/LoomGradleExtension.java b/src/main/java/net/fabricmc/loom/LoomGradleExtension.java index 3fe00fe..a01ae9f 100644 --- a/src/main/java/net/fabricmc/loom/LoomGradleExtension.java +++ b/src/main/java/net/fabricmc/loom/LoomGradleExtension.java @@ -57,7 +57,6 @@ public class LoomGradleExtension { public String loaderLaunchMethod; public boolean remapMod = true; public boolean autoGenIDERuns = true; - public boolean extractJars = false; public String customManifest = null; public File accessWidener = null; public Function intermediaryUrl = mcVer -> "https://maven.fabricmc.net/net/fabricmc/intermediary/" + mcVer + "/intermediary-" + mcVer + "-v2.jar"; diff --git a/src/main/java/net/fabricmc/loom/processors/dependency/ModDependencyInfo.java b/src/main/java/net/fabricmc/loom/processors/dependency/ModDependencyInfo.java new file mode 100644 index 0000000..92e62b7 --- /dev/null +++ b/src/main/java/net/fabricmc/loom/processors/dependency/ModDependencyInfo.java @@ -0,0 +1,104 @@ +/* + * This file is part of fabric-loom, licensed under the MIT License (MIT). + * + * Copyright (c) 2016, 2017, 2018 FabricMC + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package net.fabricmc.loom.processors.dependency; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; + +import com.google.gson.JsonObject; + +import net.fabricmc.loom.util.ModProcessor; + +public class ModDependencyInfo { + public final String group; + public final String name; + public final String version; + public final String classifier; + public final File inputFile; + + public final RemapData remapData; + + public ModDependencyInfo(String group, String name, String version, String classifier, File inputFile, RemapData remapData) { + this.group = group; + this.name = name; + this.version = version; + this.classifier = classifier; + this.inputFile = inputFile; + this.remapData = remapData; + } + + public String getRemappedNotation() { + return String.format("%s:%s:%s@%s%s", group, name, version, remapData.mappingsSuffix, classifier); + } + + public String getRemappedFilename() { + return String.format("%s-%s@%s", name, version, remapData.mappingsSuffix + classifier.replace(':', '-')); + } + + public File getRemappedOutput() { + return new File(remapData.modStore, getRemappedFilename() + ".jar"); + } + + public File getInputFile() { + return inputFile; + } + + public boolean requiresRemapping() { + return !getRemappedOutput().exists() || inputFile.lastModified() <= 0 || inputFile.lastModified() > getRemappedOutput().lastModified(); + } + + public void finaliseRemapping() { + getRemappedOutput().setLastModified(inputFile.lastModified()); + } + + @Override + public String toString() { + return String.format("%s:%s:%s:%s", group, name, version, classifier); + } + + public String getAccessWidener() throws IOException { + try (JarFile jarFile = new JarFile(getInputFile())) { + JarEntry modJsonEntry = jarFile.getJarEntry("fabric.mod.json"); + + if (modJsonEntry == null) { + return null; + } + + try (InputStream inputStream = jarFile.getInputStream(modJsonEntry)) { + JsonObject json = ModProcessor.GSON.fromJson(new InputStreamReader(inputStream), JsonObject.class); + + if (!json.has("accessWidener")) { + return null; + } + + return json.get("accessWidener").getAsString(); + } + } + } +} diff --git a/src/main/java/net/fabricmc/loom/processors/dependency/RemapData.java b/src/main/java/net/fabricmc/loom/processors/dependency/RemapData.java new file mode 100644 index 0000000..7854278 --- /dev/null +++ b/src/main/java/net/fabricmc/loom/processors/dependency/RemapData.java @@ -0,0 +1,37 @@ +/* + * This file is part of fabric-loom, licensed under the MIT License (MIT). + * + * Copyright (c) 2016, 2017, 2018 FabricMC + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package net.fabricmc.loom.processors.dependency; + +import java.io.File; + +public class RemapData { + public final String mappingsSuffix; + public final File modStore; + + public RemapData(String mappingsSuffix, File modStore) { + this.mappingsSuffix = mappingsSuffix; + this.modStore = modStore; + } +} diff --git a/src/main/java/net/fabricmc/loom/providers/MinecraftMappedProvider.java b/src/main/java/net/fabricmc/loom/providers/MinecraftMappedProvider.java index d88a162..52f9f9d 100644 --- a/src/main/java/net/fabricmc/loom/providers/MinecraftMappedProvider.java +++ b/src/main/java/net/fabricmc/loom/providers/MinecraftMappedProvider.java @@ -25,14 +25,19 @@ package net.fabricmc.loom.providers; import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.Arrays; import java.util.Collection; import java.util.function.Consumer; import org.gradle.api.Project; +import net.fabricmc.loom.util.TinyRemapperMappingsHelper; +import net.fabricmc.tinyremapper.OutputConsumerPath; +import net.fabricmc.tinyremapper.TinyRemapper; import net.fabricmc.loom.util.Constants; import net.fabricmc.loom.util.DependencyProvider; -import net.fabricmc.loom.util.MapJarsTiny; public class MinecraftMappedProvider extends DependencyProvider { private File minecraftMappedJar; @@ -66,7 +71,7 @@ public class MinecraftMappedProvider extends DependencyProvider { } try { - new MapJarsTiny().mapJars(minecraftProvider, this, this.minecraftMappedJar, this.minecraftIntermediaryJar, getProject()); + mapMinecraftJar(); } catch (Throwable t) { //Cleanup some some things that may be in a bad state now minecraftMappedJar.delete(); @@ -83,6 +88,47 @@ public class MinecraftMappedProvider extends DependencyProvider { addDependencies(dependency, postPopulationScheduler); } + private void mapMinecraftJar() throws IOException { + String fromM = "official"; + + MappingsProvider mappingsProvider = getExtension().getMappingsProvider(); + + Path input = minecraftProvider.getMergedJar().toPath(); + Path outputMapped = minecraftMappedJar.toPath(); + Path outputIntermediary = minecraftIntermediaryJar.toPath(); + + for (String toM : Arrays.asList("named", "intermediary")) { + Path output = "named".equals(toM) ? outputMapped : outputIntermediary; + + getProject().getLogger().lifecycle(":remapping minecraft (TinyRemapper, " + fromM + " -> " + toM + ")"); + + TinyRemapper remapper = getTinyRemapper(fromM, toM); + + try (OutputConsumerPath outputConsumer = new OutputConsumerPath.Builder(output).build()) { + outputConsumer.addNonClassFiles(input); + remapper.readClassPath(getRemapClasspath()); + remapper.readInputs(input); + remapper.apply(outputConsumer); + } catch (Exception e) { + throw new RuntimeException("Failed to remap JAR " + input + " with mappings from " + mappingsProvider.tinyMappings, e); + } finally { + remapper.finish(); + } + } + } + + public TinyRemapper getTinyRemapper(String fromM, String toM) throws IOException { + return TinyRemapper.newRemapper() + .withMappings(TinyRemapperMappingsHelper.create(getExtension().getMappingsProvider().getMappings(), fromM, toM, true)) + .renameInvalidLocals(true) + .rebuildSourceFilenames(true) + .build(); + } + + public Path[] getRemapClasspath() { + return getMapperPaths().stream().map(File::toPath).toArray(Path[]::new); + } + protected void addDependencies(DependencyInfo dependency, Consumer postPopulationScheduler) { getProject().getRepositories().flatDir(repository -> repository.dir(getJarDirectory(getExtension().getUserCache(), "mapped"))); diff --git a/src/main/java/net/fabricmc/loom/task/RemapJarTask.java b/src/main/java/net/fabricmc/loom/task/RemapJarTask.java index 33c6b70..6a041a1 100644 --- a/src/main/java/net/fabricmc/loom/task/RemapJarTask.java +++ b/src/main/java/net/fabricmc/loom/task/RemapJarTask.java @@ -110,11 +110,16 @@ public class RemapJarTask extends Jar { remapper.readInputs(input); remapper.apply(outputConsumer); } catch (Exception e) { - throw new RuntimeException("Failed to remap " + input + " to " + output, e); - } finally { remapper.finish(); + throw new RuntimeException("Failed to remap " + input + " to " + output, e); } + if (extension.accessWidener != null) { + extension.getJarProcessorManager().getByType(AccessWidenerJarProcessor.class).remapAccessWidener(output, remapper.getRemapper()); + } + + remapper.finish(); + if (!Files.exists(output)) { throw new RuntimeException("Failed to remap " + input + " to " + output + " - file missing!"); } @@ -129,10 +134,6 @@ public class RemapJarTask extends Jar { } } - if (extension.accessWidener != null) { - extension.getJarProcessorManager().getByType(AccessWidenerJarProcessor.class).remapAccessWidener(output); - } - /*try { if (modJar.exists()) { Files.move(modJar, modJarUnmappedCopy); diff --git a/src/main/java/net/fabricmc/loom/util/LoomDependencyManager.java b/src/main/java/net/fabricmc/loom/util/LoomDependencyManager.java index a8fb1a0..a2b0e8e 100644 --- a/src/main/java/net/fabricmc/loom/util/LoomDependencyManager.java +++ b/src/main/java/net/fabricmc/loom/util/LoomDependencyManager.java @@ -116,11 +116,13 @@ public class LoomDependencyManager { }); } + SourceRemapper sourceRemapper = new SourceRemapper(project, true); + { String mappingsKey = mappingsProvider.mappingsName + "." + mappingsProvider.minecraftVersion.replace(' ', '_').replace('.', '_').replace('-', '_') + "." + mappingsProvider.mappingsVersion; for (RemappedConfigurationEntry entry : Constants.MOD_COMPILE_ENTRIES) { - ModCompileRemapper.remapDependencies(project, mappingsKey, extension, project.getConfigurations().getByName(entry.getSourceConfiguration()), project.getConfigurations().getByName(entry.getRemappedConfiguration()), project.getConfigurations().getByName(entry.getTargetConfiguration(project.getConfigurations())), afterTasks::add); + ModCompileRemapper.remapDependencies(project, mappingsKey, extension, project.getConfigurations().getByName(entry.getSourceConfiguration()), project.getConfigurations().getByName(entry.getRemappedConfiguration()), project.getConfigurations().getByName(entry.getTargetConfiguration(project.getConfigurations())), sourceRemapper); } } @@ -150,6 +152,8 @@ public class LoomDependencyManager { project.getLogger().warn("fabric-installer.json not found in classpath!"); } + sourceRemapper.remapAll(); + for (Runnable runnable : afterTasks) { runnable.run(); } diff --git a/src/main/java/net/fabricmc/loom/util/MapJarsTiny.java b/src/main/java/net/fabricmc/loom/util/MapJarsTiny.java deleted file mode 100644 index bac57f4..0000000 --- a/src/main/java/net/fabricmc/loom/util/MapJarsTiny.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * This file is part of fabric-loom, licensed under the MIT License (MIT). - * - * Copyright (c) 2016, 2017, 2018 FabricMC - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package net.fabricmc.loom.util; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.util.Arrays; - -import org.gradle.api.Project; - -import net.fabricmc.loom.LoomGradleExtension; -import net.fabricmc.loom.providers.MappingsProvider; -import net.fabricmc.loom.providers.MinecraftMappedProvider; -import net.fabricmc.loom.providers.MinecraftProvider; -import net.fabricmc.tinyremapper.OutputConsumerPath; -import net.fabricmc.tinyremapper.TinyRemapper; - -public class MapJarsTiny { - public void mapJars(MinecraftProvider jarProvider, MinecraftMappedProvider mapProvider, File mappedJar, File intermediaryJar, Project project) throws IOException { - String fromM = "official"; - - LoomGradleExtension extension = project.getExtensions().getByType(LoomGradleExtension.class); - MappingsProvider mappingsProvider = extension.getMappingsProvider(); - - Path[] classpath = mapProvider.getMapperPaths().stream().map(File::toPath).toArray(Path[]::new); - - Path input = jarProvider.getMergedJar().toPath(); - Path outputMapped = mappedJar.toPath(); - Path outputIntermediary = intermediaryJar.toPath(); - - for (String toM : Arrays.asList("named", "intermediary")) { - Path output = "named".equals(toM) ? outputMapped : outputIntermediary; - - project.getLogger().lifecycle(":remapping minecraft (TinyRemapper, " + fromM + " -> " + toM + ")"); - - TinyRemapper remapper = TinyRemapper.newRemapper() - .withMappings(TinyRemapperMappingsHelper.create(mappingsProvider.getMappings(), fromM, toM, true)) - .renameInvalidLocals(true) - .rebuildSourceFilenames(true) - .build(); - - try (OutputConsumerPath outputConsumer = new OutputConsumerPath.Builder(output).build()) { - outputConsumer.addNonClassFiles(input); - remapper.readClassPath(classpath); - remapper.readInputs(input); - remapper.apply(outputConsumer); - } catch (Exception e) { - throw new RuntimeException("Failed to remap JAR " + input + " with mappings from " + mappingsProvider.tinyMappings, e); - } finally { - remapper.finish(); - } - } - } -} diff --git a/src/main/java/net/fabricmc/loom/util/ModCompileRemapper.java b/src/main/java/net/fabricmc/loom/util/ModCompileRemapper.java index 6f8f930..df05427 100644 --- a/src/main/java/net/fabricmc/loom/util/ModCompileRemapper.java +++ b/src/main/java/net/fabricmc/loom/util/ModCompileRemapper.java @@ -26,7 +26,9 @@ package net.fabricmc.loom.util; import java.io.File; import java.io.IOException; -import java.util.function.Consumer; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; import java.util.zip.ZipFile; import org.gradle.api.Project; @@ -45,12 +47,19 @@ import org.gradle.jvm.JvmLibrary; import org.gradle.language.base.artifact.SourcesArtifact; import net.fabricmc.loom.LoomGradleExtension; +import net.fabricmc.loom.processors.dependency.ModDependencyInfo; +import net.fabricmc.loom.processors.dependency.RemapData; public class ModCompileRemapper { - public static void remapDependencies(Project project, String mappingsSuffix, LoomGradleExtension extension, Configuration modCompile, Configuration modCompileRemapped, Configuration regularCompile, Consumer postPopulationScheduler) { + public static void remapDependencies(Project project, String mappingsSuffix, LoomGradleExtension extension, Configuration modCompile, Configuration modCompileRemapped, Configuration regularCompile, SourceRemapper sourceRemapper) { Logger logger = project.getLogger(); DependencyHandler dependencies = project.getDependencies(); + final File modStore = extension.getRemappedModCache(); + final RemapData remapData = new RemapData(mappingsSuffix, modStore); + + final List modDependencies = new ArrayList<>(); + for (ResolvedArtifact artifact : modCompile.getResolvedConfiguration().getResolvedArtifacts()) { String group; String name; @@ -76,21 +85,32 @@ public class ModCompileRemapper { File sources = findSources(dependencies, artifact); + ModDependencyInfo info = new ModDependencyInfo(group, name, version, classifierSuffix, artifact.getFile(), remapData); + modDependencies.add(info); + String remappedLog = group + ":" + name + ":" + version + classifierSuffix + " (" + mappingsSuffix + ")"; - String remappedNotation = String.format("%s:%s:%s@%s%s", group, name, version, mappingsSuffix, classifierSuffix); String remappedFilename = String.format("%s-%s@%s", name, version, mappingsSuffix + classifierSuffix.replace(':', '-')); project.getLogger().info(":providing " + remappedLog); - File modStore = extension.getRemappedModCache(); - - remapArtifact(project, modCompileRemapped, artifact, remappedFilename, modStore); - - project.getDependencies().add(modCompileRemapped.getName(), project.getDependencies().module(remappedNotation)); - if (sources != null) { - scheduleSourcesRemapping(project, postPopulationScheduler, sources, remappedLog, remappedFilename, modStore); + scheduleSourcesRemapping(project, sourceRemapper, sources, remappedLog, remappedFilename, modStore); } } + + List processList = modDependencies.stream() + .filter(ModDependencyInfo::requiresRemapping) + .collect(Collectors.toList()); + + try { + ModProcessor.processMods(project, processList); + } catch (IOException e) { + throw new RuntimeException("Failed to remap mods", e); + } + + // Add all of the remapped mods onto the config + for (ModDependencyInfo modDependency : modDependencies) { + project.getDependencies().add(modCompileRemapped.getName(), project.getDependencies().module(modDependency.getRemappedNotation())); + } } /** @@ -123,28 +143,6 @@ public class ModCompileRemapper { dependencies.add(regularCompile.getName(), dep); } - private static void remapArtifact(Project project, Configuration config, ResolvedArtifact artifact, String remappedFilename, File modStore) { - File input = artifact.getFile(); - File output = new File(modStore, remappedFilename + ".jar"); - - if (!output.exists() || input.lastModified() <= 0 || input.lastModified() > output.lastModified()) { - //If the output doesn't exist, or appears to be outdated compared to the input we'll remap it - try { - ModProcessor.processMod(input, output, project, config, artifact); - } catch (IOException e) { - throw new RuntimeException("Failed to remap mod", e); - } - - if (!output.exists()) { - throw new RuntimeException("Failed to remap mod"); - } - - output.setLastModified(input.lastModified()); - } else { - project.getLogger().info(output.getName() + " is up to date with " + input.getName()); - } - } - public static File findSources(DependencyHandler dependencies, ResolvedArtifact artifact) { @SuppressWarnings("unchecked") ArtifactResolutionQuery query = dependencies.createArtifactResolutionQuery()// .forComponents(artifact.getId().getComponentIdentifier())// @@ -161,23 +159,21 @@ public class ModCompileRemapper { return null; } - private static void scheduleSourcesRemapping(Project project, Consumer postPopulationScheduler, File sources, String remappedLog, String remappedFilename, File modStore) { - postPopulationScheduler.accept(() -> { - project.getLogger().info(":providing " + remappedLog + " sources"); - File remappedSources = new File(modStore, remappedFilename + "-sources.jar"); + private static void scheduleSourcesRemapping(Project project, SourceRemapper sourceRemapper, File sources, String remappedLog, String remappedFilename, File modStore) { + project.getLogger().info(":providing " + remappedLog + " sources"); + File remappedSources = new File(modStore, remappedFilename + "-sources.jar"); - if (!remappedSources.exists() || sources.lastModified() <= 0 || sources.lastModified() > remappedSources.lastModified()) { - try { - SourceRemapper.remapSources(project, sources, remappedSources, true); + if (!remappedSources.exists() || sources.lastModified() <= 0 || sources.lastModified() > remappedSources.lastModified()) { + try { + sourceRemapper.scheduleRemapSources(sources, remappedSources); - //Set the remapped sources creation date to match the sources if we're likely succeeded in making it - remappedSources.setLastModified(sources.lastModified()); - } catch (Exception e) { - e.printStackTrace(); - } - } else { - project.getLogger().info(remappedSources.getName() + " is up to date with " + sources.getName()); + //Set the remapped sources creation date to match the sources if we're likely succeeded in making it + remappedSources.setLastModified(sources.lastModified()); + } catch (Exception e) { + e.printStackTrace(); } - }); + } else { + project.getLogger().info(remappedSources.getName() + " is up to date with " + sources.getName()); + } } } diff --git a/src/main/java/net/fabricmc/loom/util/ModProcessor.java b/src/main/java/net/fabricmc/loom/util/ModProcessor.java index 9d5b3d4..cb9d53f 100644 --- a/src/main/java/net/fabricmc/loom/util/ModProcessor.java +++ b/src/main/java/net/fabricmc/loom/util/ModProcessor.java @@ -25,31 +25,28 @@ package net.fabricmc.loom.util; import java.io.BufferedReader; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.io.StringReader; import java.io.StringWriter; import java.nio.charset.StandardCharsets; import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.HashSet; -import java.util.Set; -import java.util.jar.JarEntry; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.BiConsumer; import java.util.jar.JarFile; import java.util.zip.ZipEntry; import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import com.google.gson.JsonArray; import com.google.gson.JsonObject; import org.apache.commons.io.IOUtils; import org.gradle.api.Project; -import org.gradle.api.artifacts.Configuration; -import org.gradle.api.artifacts.ResolvedArtifact; +import org.objectweb.asm.commons.Remapper; import org.zeroturnaround.zip.ZipUtil; -import org.zeroturnaround.zip.commons.FileUtils; import org.zeroturnaround.zip.transform.StringZipEntryTransformer; import org.zeroturnaround.zip.transform.ZipEntryTransformerEntry; @@ -58,82 +55,34 @@ import net.fabricmc.loom.providers.MappingsProvider; import net.fabricmc.loom.providers.MinecraftMappedProvider; import net.fabricmc.loom.util.accesswidener.AccessWidener; import net.fabricmc.loom.util.accesswidener.AccessWidenerRemapper; -import net.fabricmc.tinyremapper.OutputConsumerPath; +import net.fabricmc.loom.processors.dependency.ModDependencyInfo; import net.fabricmc.tinyremapper.TinyRemapper; +import net.fabricmc.tinyremapper.InputTag; +import net.fabricmc.tinyremapper.OutputConsumerPath; public class ModProcessor { - private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + public static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); - public static void processMod(File input, File output, Project project, Configuration config, ResolvedArtifact artifact) throws IOException { - if (output.exists()) { - output.delete(); + public static void processMods(Project project, List processList) throws IOException { + if (processList.isEmpty()) { + return; } - remapJar(input, output, project, artifact); - - //Enable this if you want your nested jars to be extracted, this will extract **all** jars - if (project.getExtensions().getByType(LoomGradleExtension.class).extractJars) { - handleNestedJars(input, project, config, artifact); - } - - remapaccessWidener(output, project); - - //Always strip the nested jars - stripNestedJars(output); - } - - private static void handleNestedJars(File input, Project project, Configuration config, ResolvedArtifact artifact) throws IOException { - try (JarFile jarFile = new JarFile(input)) { - JarEntry modJsonEntry = jarFile.getJarEntry("fabric.mod.json"); - - if (modJsonEntry == null) { - return; - } - - try (InputStream inputStream = jarFile.getInputStream(modJsonEntry)) { - JsonObject json = GSON.fromJson(new InputStreamReader(inputStream), JsonObject.class); - - if (json == null || !json.has("jars")) { - return; - } - - JsonArray jsonArray = json.getAsJsonArray("jars"); - - for (int i = 0; i < jsonArray.size(); i++) { - JsonObject jsonObject = jsonArray.get(i).getAsJsonObject(); - String fileName = jsonObject.get("file").getAsString(); - project.getLogger().lifecycle(String.format("Found %s nested in %s", fileName, input.getName())); - processNestedJar(jarFile, fileName, project, config, artifact); - } + for (ModDependencyInfo info : processList) { + if (info.getRemappedOutput().exists()) { + info.getRemappedOutput().delete(); } } - } - private static void processNestedJar(JarFile parentJar, String fileName, Project project, Configuration config, ResolvedArtifact artifact) throws IOException { - LoomGradleExtension extension = project.getExtensions().getByType(LoomGradleExtension.class); + remapJars(project, processList); - JarEntry entry = parentJar.getJarEntry(fileName); + for (ModDependencyInfo info : processList) { + if (!info.getRemappedOutput().exists()) { + throw new RuntimeException("Failed to remap mod" + info); + } - if (entry == null) { - throw new RuntimeException(String.format("%s was not found in %s", fileName, parentJar.getName())); + stripNestedJars(info.getRemappedOutput()); } - - File nestedFile = new File(extension.getNestedModCache(), fileName.substring(fileName.lastIndexOf("/"))); - - try (InputStream jarStream = parentJar.getInputStream(entry)) { - FileUtils.copy(jarStream, nestedFile); - } - - File remappedFile = new File(extension.getRemappedModCache(), fileName.substring(fileName.lastIndexOf("/"))); - - processMod(nestedFile, remappedFile, project, config, artifact); - - if (!remappedFile.exists()) { - throw new RuntimeException("Failed to find processed nested jar"); - } - - //Add the project right onto the remapped mods, hopefully this works - project.getDependencies().add(config.getName(), project.files(remappedFile)); } private static void stripNestedJars(File file) { @@ -148,57 +97,24 @@ public class ModProcessor { }))}); } - private static void remapaccessWidener(File input, Project project) throws IOException { - String accessWidenerPath; - - try (JarFile jarFile = new JarFile(input)) { - JarEntry modJsonEntry = jarFile.getJarEntry("fabric.mod.json"); - - if (modJsonEntry == null) { - return; - } - - try (InputStream inputStream = jarFile.getInputStream(modJsonEntry)) { - JsonObject json = GSON.fromJson(new InputStreamReader(inputStream), JsonObject.class); - - if (!json.has("accessWidener")) { - return; - } - - accessWidenerPath = json.get("accessWidener").getAsString(); - } - } - - if (accessWidenerPath == null) { - return; - } - - ZipUtil.transformEntry(input, accessWidenerPath, new StringZipEntryTransformer() { - @Override - protected String transform(ZipEntry zipEntry, String input) throws IOException { - return remapaccessWidener(input, project); - } - }); - } - - private static String remapaccessWidener(String input, Project project) { - try (BufferedReader bufferedReader = new BufferedReader(new StringReader(input))) { + private static byte[] remapaccessWidener(byte[] input, Remapper remapper) { + try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(input), StandardCharsets.UTF_8))) { AccessWidener accessWidener = new AccessWidener(); accessWidener.read(bufferedReader); - AccessWidenerRemapper accessWidenerRemapper = new AccessWidenerRemapper(accessWidener, project.getExtensions().getByType(LoomGradleExtension.class).getMappingsProvider().getMappings(), "named"); + AccessWidenerRemapper accessWidenerRemapper = new AccessWidenerRemapper(accessWidener, remapper, "named"); AccessWidener remapped = accessWidenerRemapper.remap(); try (StringWriter writer = new StringWriter()) { remapped.write(writer); - return writer.toString(); + return writer.toString().getBytes(StandardCharsets.UTF_8); } } catch (IOException e) { throw new RuntimeException(e); } } - private static void remapJar(File input, File output, Project project, ResolvedArtifact artifact) throws IOException { + private static void remapJars(Project project, List processList) throws IOException { LoomGradleExtension extension = project.getExtensions().getByType(LoomGradleExtension.class); String fromM = "intermediary"; String toM = "named"; @@ -206,47 +122,57 @@ public class ModProcessor { MinecraftMappedProvider mappedProvider = extension.getMinecraftMappedProvider(); MappingsProvider mappingsProvider = extension.getMappingsProvider(); - Path inputPath = input.getAbsoluteFile().toPath(); Path mc = mappedProvider.getIntermediaryJar().toPath(); Path[] mcDeps = mappedProvider.getMapperPaths().stream().map(File::toPath).toArray(Path[]::new); - Set modCompiles = new HashSet<>(); - for (RemappedConfigurationEntry entry : Constants.MOD_COMPILE_ENTRIES) { - project.getConfigurations().getByName(entry.getSourceConfiguration()).getFiles().stream().filter((f) -> !f.equals(input)).map(p -> { - if (p.equals(input)) { - return inputPath; - } else { - return p.toPath(); - } - }).forEach(modCompiles::add); - } - - project.getLogger().lifecycle(":remapping " + input.getName() + " (TinyRemapper, " + fromM + " -> " + toM + ")"); - - // If the sources don't exist, we want remapper to give nicer names to the missing variable names. - // However, if the sources do exist, if remapper gives names to the parameters that prevents IDEs (at least IDEA) - // from replacing the parameters with the actual names from the sources. - boolean sourcesExist = ModCompileRemapper.findSources(project.getDependencies(), artifact) != null; + project.getLogger().lifecycle(":remapping " + processList.size() + " mods (TinyRemapper, " + fromM + " -> " + toM + ")"); TinyRemapper remapper = TinyRemapper.newRemapper() .withMappings(TinyRemapperMappingsHelper.create(mappingsProvider.getMappings(), fromM, toM, false)) - .renameInvalidLocals(!sourcesExist) + .renameInvalidLocals(false) .build(); - try (OutputConsumerPath outputConsumer = new OutputConsumerPath.Builder(Paths.get(output.getAbsolutePath())).build()) { - outputConsumer.addNonClassFiles(inputPath); - remapper.readClassPath(modCompiles.toArray(new Path[0])); - remapper.readClassPath(mc); - remapper.readClassPath(mcDeps); - remapper.readInputs(inputPath); - remapper.apply(outputConsumer); - } finally { - remapper.finish(); + remapper.readClassPathAsync(mc); + remapper.readClassPathAsync(mcDeps); + + final Map tagMap = new HashMap<>(); + final Map outputConsumerMap = new HashMap<>(); + + for (ModDependencyInfo info : processList) { + InputTag tag = remapper.createInputTag(); + remapper.readInputsAsync(tag, info.getInputFile().toPath()); + tagMap.put(info, tag); } - if (!output.exists()) { - throw new RuntimeException("Failed to remap JAR to " + toM + " file not found: " + output.getAbsolutePath()); + // Apply this in a second loop as we need to ensure all the inputs are on the classpath before remapping. + for (ModDependencyInfo info : processList) { + OutputConsumerPath outputConsumer = new OutputConsumerPath.Builder(info.getRemappedOutput().toPath()).build(); + outputConsumer.addNonClassFiles(info.getInputFile().toPath()); + outputConsumerMap.put(info, outputConsumer); + String accessWidener = info.getAccessWidener(); + + if (accessWidener == null) { + remapper.apply(outputConsumer, tagMap.get(info)); + } else { + remapper.apply(remapAccessWidener(remapper.getRemapper(), accessWidener, outputConsumer), tagMap.get(info)); + } } + + remapper.finish(); + + for (ModDependencyInfo info : processList) { + outputConsumerMap.get(info).close(); + } + } + + static BiConsumer remapAccessWidener(Remapper remapper, String accessWidener, BiConsumer output) { + return (s, bytes) -> { + if (s.equals(accessWidener)) { + output.accept(s, remapaccessWidener(bytes, remapper)); + } else { + output.accept(s, bytes); + } + }; } static JsonObject readInstallerJson(File file, Project project) { diff --git a/src/main/java/net/fabricmc/loom/util/SourceRemapper.java b/src/main/java/net/fabricmc/loom/util/SourceRemapper.java index fd22e4d..ab57907 100644 --- a/src/main/java/net/fabricmc/loom/util/SourceRemapper.java +++ b/src/main/java/net/fabricmc/loom/util/SourceRemapper.java @@ -28,6 +28,8 @@ import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; import org.cadixdev.lorenz.MappingSet; import org.cadixdev.lorenz.io.MappingsReader; @@ -46,44 +48,46 @@ import net.fabricmc.mapping.tree.TinyTree; import net.fabricmc.stitch.util.StitchUtil; public class SourceRemapper { - public static void remapSources(Project project, File source, File destination, boolean toNamed) throws Exception { - remapSourcesInner(project, source, destination, toNamed); + private final Project project; + private final boolean toNamed; + private final List remapTasks = new ArrayList<>(); + + private Mercury mercury; + + public SourceRemapper(Project project, boolean toNamed) { + this.project = project; + this.toNamed = toNamed; + } + + public static void remapSources(Project project, File input, File output, boolean named) throws Exception { + SourceRemapper sourceRemapper = new SourceRemapper(project, named); + sourceRemapper.scheduleRemapSources(input, output); + sourceRemapper.remapAll(); + } + + public void scheduleRemapSources(File source, File destination) throws Exception { + remapTasks.add(() -> { + try { + remapSourcesInner(source, destination); + } catch (Exception e) { + throw new RuntimeException("Failed to remap sources for " + source, e); + } + }); + } + + public void remapAll() { + if (!remapTasks.isEmpty()) { + project.getLogger().lifecycle(":remapping sources"); + } + + remapTasks.forEach(Runnable::run); // TODO: FIXME - WORKAROUND https://github.com/FabricMC/fabric-loom/issues/45 System.gc(); } - private static void remapSourcesInner(Project project, File source, File destination, boolean toNamed) throws Exception { - LoomGradleExtension extension = project.getExtensions().getByType(LoomGradleExtension.class); - MappingsProvider mappingsProvider = extension.getMappingsProvider(); - - MappingSet mappings = extension.getOrCreateSrcMappingCache(toNamed ? 1 : 0, () -> { - try { - TinyTree m = mappingsProvider.getMappings(); - project.getLogger().lifecycle(":loading " + (toNamed ? "intermediary -> named" : "named -> intermediary") + " source mappings"); - return new TinyReader(m, toNamed ? "intermediary" : "named", toNamed ? "named" : "intermediary").read(); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - + private void remapSourcesInner(File source, File destination) throws Exception { project.getLogger().info(":remapping source jar"); - - Mercury mercury = extension.getOrCreateSrcMercuryCache(toNamed ? 1 : 0, () -> { - Mercury m = createMercuryWithClassPath(project, toNamed); - - for (Path file : extension.getUnmappedMods()) { - if (Files.isRegularFile(file)) { - m.getClassPath().add(file); - } - } - - m.getClassPath().add(extension.getMinecraftMappedProvider().getMappedJar().toPath()); - m.getClassPath().add(extension.getMinecraftMappedProvider().getIntermediaryJar().toPath()); - - m.getProcessors().add(MercuryRemapper.create(mappings)); - - return m; - }); + Mercury mercury = getMercuryInstance(); if (source.equals(destination)) { if (source.isDirectory()) { @@ -135,6 +139,45 @@ public class SourceRemapper { } } + private Mercury getMercuryInstance() { + if (this.mercury != null) { + return this.mercury; + } + + LoomGradleExtension extension = project.getExtensions().getByType(LoomGradleExtension.class); + MappingsProvider mappingsProvider = extension.getMappingsProvider(); + + MappingSet mappings = extension.getOrCreateSrcMappingCache(toNamed ? 1 : 0, () -> { + try { + TinyTree m = mappingsProvider.getMappings(); + project.getLogger().lifecycle(":loading " + (toNamed ? "intermediary -> named" : "named -> intermediary") + " source mappings"); + return new TinyReader(m, toNamed ? "intermediary" : "named", toNamed ? "named" : "intermediary").read(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + Mercury mercury = extension.getOrCreateSrcMercuryCache(toNamed ? 1 : 0, () -> { + Mercury m = createMercuryWithClassPath(project, toNamed); + + for (Path file : extension.getUnmappedMods()) { + if (Files.isRegularFile(file)) { + m.getClassPath().add(file); + } + } + + m.getClassPath().add(extension.getMinecraftMappedProvider().getMappedJar().toPath()); + m.getClassPath().add(extension.getMinecraftMappedProvider().getIntermediaryJar().toPath()); + + m.getProcessors().add(MercuryRemapper.create(mappings)); + + return m; + }); + + this.mercury = mercury; + return mercury; + } + private static void copyNonJavaFiles(Path from, Path to, Project project, File source) throws IOException { Files.walk(from).forEach(path -> { Path targetPath = to.resolve(from.relativize(path).toString()); diff --git a/src/main/java/net/fabricmc/loom/util/accesswidener/AccessWidenerJarProcessor.java b/src/main/java/net/fabricmc/loom/util/accesswidener/AccessWidenerJarProcessor.java index 9416ca9..7f1af33 100644 --- a/src/main/java/net/fabricmc/loom/util/accesswidener/AccessWidenerJarProcessor.java +++ b/src/main/java/net/fabricmc/loom/util/accesswidener/AccessWidenerJarProcessor.java @@ -44,6 +44,7 @@ import org.objectweb.asm.ClassWriter; import org.objectweb.asm.FieldVisitor; import org.objectweb.asm.MethodVisitor; import org.objectweb.asm.Opcodes; +import org.objectweb.asm.commons.Remapper; import org.zeroturnaround.zip.ZipUtil; import org.zeroturnaround.zip.transform.ByteArrayZipEntryTransformer; import org.zeroturnaround.zip.transform.ZipEntryTransformer; @@ -53,6 +54,7 @@ import net.fabricmc.mappings.EntryTriple; import net.fabricmc.loom.LoomGradleExtension; import net.fabricmc.loom.util.Checksum; import net.fabricmc.loom.processors.JarProcessor; +import net.fabricmc.tinyremapper.TinyRemapper; public class AccessWidenerJarProcessor implements JarProcessor { private AccessWidener accessWidener = new AccessWidener(); @@ -79,8 +81,13 @@ public class AccessWidenerJarProcessor implements JarProcessor { //Remap accessWidener if its not named, allows for AE's to be written in intermediary if (!accessWidener.namespace.equals("named")) { try { - AccessWidenerRemapper remapper = new AccessWidenerRemapper(accessWidener, loomGradleExtension.getMappingsProvider().getMappings(), "named"); + TinyRemapper tinyRemapper = loomGradleExtension.getMinecraftMappedProvider().getTinyRemapper("official", "named"); + tinyRemapper.readClassPath(loomGradleExtension.getMinecraftMappedProvider().getRemapClasspath()); + + AccessWidenerRemapper remapper = new AccessWidenerRemapper(accessWidener, tinyRemapper.getRemapper(), "named"); accessWidener = remapper.remap(); + + tinyRemapper.finish(); } catch (IOException e) { throw new RuntimeException("Failed to remap access widener", e); } @@ -116,9 +123,8 @@ public class AccessWidenerJarProcessor implements JarProcessor { } //Called when remapping the mod - public void remapAccessWidener(Path modJarPath) throws IOException { - LoomGradleExtension extension = project.getExtensions().getByType(LoomGradleExtension.class); - AccessWidenerRemapper remapper = new AccessWidenerRemapper(accessWidener, extension.getMappingsProvider().getMappings(), "intermediary"); + public void remapAccessWidener(Path modJarPath, Remapper asmRemapper) throws IOException { + AccessWidenerRemapper remapper = new AccessWidenerRemapper(accessWidener, asmRemapper, "intermediary"); AccessWidener remapped = remapper.remap(); StringWriter writer = new StringWriter(); diff --git a/src/main/java/net/fabricmc/loom/util/accesswidener/AccessWidenerRemapper.java b/src/main/java/net/fabricmc/loom/util/accesswidener/AccessWidenerRemapper.java index cda83d6..ceb788b 100644 --- a/src/main/java/net/fabricmc/loom/util/accesswidener/AccessWidenerRemapper.java +++ b/src/main/java/net/fabricmc/loom/util/accesswidener/AccessWidenerRemapper.java @@ -24,54 +24,21 @@ package net.fabricmc.loom.util.accesswidener; -import java.util.HashMap; import java.util.Map; -import net.fabricmc.mapping.tree.ClassDef; -import net.fabricmc.mapping.tree.FieldDef; -import net.fabricmc.mapping.tree.MethodDef; -import net.fabricmc.mapping.tree.TinyTree; +import org.objectweb.asm.commons.Remapper; + import net.fabricmc.mappings.EntryTriple; public class AccessWidenerRemapper { private final AccessWidener input; - private final String from, to; + private final String to; + private final Remapper remapper; - private Map classNames = new HashMap<>(); - private Map fieldNames = new HashMap<>(); - private Map methodNames = new HashMap<>(); - - public AccessWidenerRemapper(AccessWidener input, TinyTree tinyTree, String to) { + public AccessWidenerRemapper(AccessWidener input, Remapper remapper, String to) { this.input = input; - this.from = input.namespace; this.to = to; - populateMappings(tinyTree); - } - - private void populateMappings(TinyTree tinyTree) { - if (!tinyTree.getMetadata().getNamespaces().contains(from)) { - throw new UnsupportedOperationException("Unknown namespace: " + from); - } - - if (!tinyTree.getMetadata().getNamespaces().contains(to)) { - throw new UnsupportedOperationException("Unknown namespace: " + to); - } - - for (ClassDef classDef : tinyTree.getClasses()) { - classNames.put(classDef.getName(from), classDef.getName(to)); - - for (FieldDef fieldDef : classDef.getFields()) { - EntryTriple fromEntry = new EntryTriple(classDef.getName(from), fieldDef.getName(from), fieldDef.getDescriptor(from)); - EntryTriple toEntry = new EntryTriple(classDef.getName(to), fieldDef.getName(to), fieldDef.getDescriptor(to)); - fieldNames.put(fromEntry, toEntry); - } - - for (MethodDef methodDef : classDef.getMethods()) { - EntryTriple fromEntry = new EntryTriple(classDef.getName(from), methodDef.getName(from), methodDef.getDescriptor(from)); - EntryTriple toEntry = new EntryTriple(classDef.getName(to), methodDef.getName(to), methodDef.getDescriptor(to)); - methodNames.put(fromEntry, toEntry); - } - } + this.remapper = remapper; } public AccessWidener remap() { @@ -84,27 +51,33 @@ public class AccessWidenerRemapper { remapped.namespace = to; for (Map.Entry entry : input.classAccess.entrySet()) { - remapped.classAccess.put(findMapping(classNames, entry.getKey()), entry.getValue()); + remapped.classAccess.put(remapper.map(entry.getKey()), entry.getValue()); } for (Map.Entry entry : input.methodAccess.entrySet()) { - remapped.addOrMerge(remapped.methodAccess, findMapping(methodNames, entry.getKey()), entry.getValue()); + remapped.addOrMerge(remapped.methodAccess, remapMethod(entry.getKey()), entry.getValue()); } for (Map.Entry entry : input.fieldAccess.entrySet()) { - remapped.addOrMerge(remapped.fieldAccess, findMapping(fieldNames, entry.getKey()), entry.getValue()); + remapped.addOrMerge(remapped.fieldAccess, remapField(entry.getKey()), entry.getValue()); } return remapped; } - private static V findMapping(Map map, K key) { - V value = map.get(key); + private EntryTriple remapMethod(EntryTriple entryTriple) { + return new EntryTriple( + remapper.map(entryTriple.getName()), + remapper.mapMethodName(entryTriple.getOwner(), entryTriple.getName(), entryTriple.getDesc()), + remapper.mapDesc(entryTriple.getDesc()) + ); + } - if (value == null) { - throw new RuntimeException("Failed to find mapping for " + key.toString()); - } - - return value; + private EntryTriple remapField(EntryTriple entryTriple) { + return new EntryTriple( + remapper.map(entryTriple.getName()), + remapper.mapFieldName(entryTriple.getOwner(), entryTriple.getName(), entryTriple.getDesc()), + remapper.mapDesc(entryTriple.getDesc()) + ); } }