From 9029a699d93df18c5c845a5afc34b9dbf380f82b Mon Sep 17 00:00:00 2001 From: modmuss50 Date: Mon, 24 Aug 2020 22:12:26 +0100 Subject: [PATCH] Official Mojang Mappings support # Please read and understand the EULA before using! Use in your build.gradle with the following: `mappings minecraft.officialMojangMappings()` To automatically migrate to Mojang mappings run the following: `gradlew.bat migrateMappings --mappings "net.mojang.minecraft:mappings:1.16.2"` Co-authored-by: Ramid Khan --- .github/workflows/test-push.yml | 2 +- build.gradle | 1 + .../fabricmc/loom/LoomGradleExtension.java | 5 + .../loom/providers/MappingsProvider.java | 59 ++-- .../loom/task/MigrateMappingsTask.java | 13 +- .../loom/util/DependencyProvider.java | 4 + .../mappings/MojangMappingsDependency.java | 268 ++++++++++++++++++ 7 files changed, 331 insertions(+), 21 deletions(-) create mode 100644 src/main/java/net/fabricmc/loom/util/mappings/MojangMappingsDependency.java diff --git a/.github/workflows/test-push.yml b/.github/workflows/test-push.yml index a33c595..e0d4f3b 100644 --- a/.github/workflows/test-push.yml +++ b/.github/workflows/test-push.yml @@ -4,7 +4,7 @@ jobs: gradle: strategy: matrix: - gradle: [4.9, 4.10.2, 6.5.1] + gradle: [4.9, 4.10.2, 6.6] java: [jdk8, jdk11, jdk14] exclude: # Dont run older gradle versions on newer java - java: jdk14 diff --git a/build.gradle b/build.gradle index 0920157..5ef1a36 100644 --- a/build.gradle +++ b/build.gradle @@ -53,6 +53,7 @@ dependencies { implementation ('net.fabricmc:lorenz-tiny:2.0.0+build.2') { transitive = false } + implementation ('org.cadixdev:lorenz-io-proguard:0.5.3') // decompilers implementation ('net.fabricmc:procyon-fabric-compilertools:0.5.35.13') diff --git a/src/main/java/net/fabricmc/loom/LoomGradleExtension.java b/src/main/java/net/fabricmc/loom/LoomGradleExtension.java index a15623c..201b77f 100644 --- a/src/main/java/net/fabricmc/loom/LoomGradleExtension.java +++ b/src/main/java/net/fabricmc/loom/LoomGradleExtension.java @@ -53,6 +53,7 @@ import net.fabricmc.loom.providers.MappingsProvider; import net.fabricmc.loom.providers.MinecraftMappedProvider; import net.fabricmc.loom.providers.MinecraftProvider; import net.fabricmc.loom.util.LoomDependencyManager; +import net.fabricmc.loom.util.mappings.MojangMappingsDependency; public class LoomGradleExtension { public String runDir = "run"; @@ -105,6 +106,10 @@ public class LoomGradleExtension { return srcMercuryCache[id] != null ? srcMercuryCache[id] : (srcMercuryCache[id] = factory.get()); } + public Dependency officialMojangMappings() { + return new MojangMappingsDependency(project, this); + } + public LoomGradleExtension(Project project) { this.project = project; this.autoGenIDERuns = AbstractPlugin.isRootProject(project); diff --git a/src/main/java/net/fabricmc/loom/providers/MappingsProvider.java b/src/main/java/net/fabricmc/loom/providers/MappingsProvider.java index 21c9b1f..80c9828 100644 --- a/src/main/java/net/fabricmc/loom/providers/MappingsProvider.java +++ b/src/main/java/net/fabricmc/loom/providers/MappingsProvider.java @@ -66,8 +66,10 @@ public class MappingsProvider extends DependencyProvider { public String minecraftVersion; public String mappingsVersion; - private Path mappingsDir; - private Path mappingsStepsDir; + private final Path mappingsDir; + private final Path mappingsStepsDir; + private Path intermediaryTiny; + private boolean hasRefreshed = false; // The mappings that gradle gives us private Path baseTinyMappings; // The mappings we use in practice @@ -77,6 +79,8 @@ public class MappingsProvider extends DependencyProvider { public MappingsProvider(Project project) { super(project); + mappingsDir = getExtension().getUserCache().toPath().resolve("mappings"); + mappingsStepsDir = mappingsDir.resolve("steps"); } public void clean() throws IOException { @@ -172,14 +176,7 @@ public class MappingsProvider extends DependencyProvider { if (baseMappingsAreV2()) { // These are unmerged v2 mappings - - // Download and extract intermediary - String encodedMinecraftVersion = UrlEscapers.urlFragmentEscaper().escape(minecraftVersion); - String intermediaryArtifactUrl = getExtension().getIntermediaryUrl().apply(encodedMinecraftVersion); - Path intermediaryJar = mappingsStepsDir.resolve("v2-intermediary-" + minecraftVersion + ".jar"); - DownloadUtil.downloadIfChanged(new URL(intermediaryArtifactUrl), intermediaryJar.toFile(), project.getLogger()); - - mergeAndSaveMappings(project, intermediaryJar, yarnJar); + mergeAndSaveMappings(project, yarnJar); } else { // These are merged v1 mappings if (tinyMappings.exists()) { @@ -216,14 +213,15 @@ public class MappingsProvider extends DependencyProvider { Files.copy(jar.getPath("mappings/mappings.tiny"), extractTo, StandardCopyOption.REPLACE_EXISTING); } - private void mergeAndSaveMappings(Project project, Path unmergedIntermediaryJar, Path unmergedYarnJar) throws IOException { - Path unmergedIntermediary = Paths.get(mappingsStepsDir.toString(), "unmerged-intermediary.tiny"); - project.getLogger().info(":extracting " + unmergedIntermediaryJar.getFileName()); + private void extractIntermediary(Path intermediaryJar, Path intermediaryTiny) throws IOException { + getProject().getLogger().info(":extracting " + intermediaryJar.getFileName()); - try (FileSystem unmergedIntermediaryFs = FileSystems.newFileSystem(unmergedIntermediaryJar, (ClassLoader) null)) { - extractMappings(unmergedIntermediaryFs, unmergedIntermediary); + try (FileSystem unmergedIntermediaryFs = FileSystems.newFileSystem(intermediaryJar, (ClassLoader) null)) { + extractMappings(unmergedIntermediaryFs, intermediaryTiny); } + } + private void mergeAndSaveMappings(Project project, Path unmergedYarnJar) throws IOException { Path unmergedYarn = Paths.get(mappingsStepsDir.toString(), "unmerged-yarn.tiny"); project.getLogger().info(":extracting " + unmergedYarnJar.getFileName()); @@ -232,7 +230,7 @@ public class MappingsProvider extends DependencyProvider { } Path invertedIntermediary = Paths.get(mappingsStepsDir.toString(), "inverted-intermediary.tiny"); - reorderMappings(unmergedIntermediary, invertedIntermediary, "intermediary", "official"); + reorderMappings(getIntermediaryTiny(), invertedIntermediary, "intermediary", "official"); Path unorderedMergedMappings = Paths.get(mappingsStepsDir.toString(), "unordered-merged.tiny"); project.getLogger().info(":merging"); mergeMappings(invertedIntermediary, unmergedYarn, unorderedMergedMappings); @@ -277,9 +275,6 @@ public class MappingsProvider extends DependencyProvider { } private void initFiles() { - mappingsDir = getExtension().getUserCache().toPath().resolve("mappings"); - mappingsStepsDir = mappingsDir.resolve("steps"); - baseTinyMappings = mappingsDir.resolve(mappingsName + "-tiny-" + minecraftVersion + "-" + mappingsVersion + "-base"); mappingsMixinExport = new File(getExtension().getProjectBuildCache(), "mixin-map-" + minecraftVersion + "-" + mappingsVersion + ".tiny"); } @@ -312,4 +307,30 @@ public class MappingsProvider extends DependencyProvider { public String getTargetConfig() { return Constants.MAPPINGS; } + + public Path getMappingsDir() { + return mappingsDir; + } + + public Path getIntermediaryTiny() throws IOException { + if (intermediaryTiny == null) { + intermediaryTiny = Paths.get(mappingsStepsDir.toString(), String.format("intermediary-%s%s.tiny", minecraftVersion, mappingsVersion)); + + if (!Files.exists(intermediaryTiny) || (isRefreshDeps() && !hasRefreshed)) { + hasRefreshed = true; + + minecraftVersion = getExtension().getMinecraftProvider().getMinecraftVersion(); + + // Download and extract intermediary + String encodedMinecraftVersion = UrlEscapers.urlFragmentEscaper().escape(minecraftVersion); + String intermediaryArtifactUrl = getExtension().getIntermediaryUrl().apply(encodedMinecraftVersion); + Path intermediaryJar = mappingsDir.resolve("v2-intermediary-" + minecraftVersion + ".jar"); + DownloadUtil.downloadIfChanged(new URL(intermediaryArtifactUrl), intermediaryJar.toFile(), getProject().getLogger()); + + extractIntermediary(intermediaryJar, intermediaryTiny); + } + } + + return intermediaryTiny; + } } diff --git a/src/main/java/net/fabricmc/loom/task/MigrateMappingsTask.java b/src/main/java/net/fabricmc/loom/task/MigrateMappingsTask.java index a048d5a..4ec85e1 100644 --- a/src/main/java/net/fabricmc/loom/task/MigrateMappingsTask.java +++ b/src/main/java/net/fabricmc/loom/task/MigrateMappingsTask.java @@ -43,6 +43,7 @@ import org.gradle.api.GradleException; import org.gradle.api.IllegalDependencyNotation; import org.gradle.api.JavaVersion; import org.gradle.api.Project; +import org.gradle.api.artifacts.Dependency; import org.gradle.api.plugins.JavaPluginConvention; import org.gradle.api.tasks.TaskAction; import org.gradle.api.tasks.options.Option; @@ -51,6 +52,7 @@ import net.fabricmc.loom.LoomGradleExtension; import net.fabricmc.loom.providers.MappingsProvider; import net.fabricmc.loom.providers.MinecraftMappedProvider; import net.fabricmc.loom.util.SourceRemapper; +import net.fabricmc.loom.util.mappings.MojangMappingsDependency; import net.fabricmc.lorenztiny.TinyMappingsJoiner; import net.fabricmc.mapping.tree.TinyMappingFactory; import net.fabricmc.mapping.tree.TinyTree; @@ -116,7 +118,16 @@ public class MigrateMappingsTask extends AbstractLoomTask { Set files; try { - files = project.getConfigurations().detachedConfiguration(project.getDependencies().create(mappings)).resolve(); + if (mappings.startsWith("net.mojang.minecraft:mappings:")) { + if (!mappings.endsWith(":" + project.getExtensions().getByType(LoomGradleExtension.class).getMinecraftProvider().getMinecraftVersion())) { + throw new UnsupportedOperationException("Migrating Mojang mappings is currently only supported for the specified minecraft version"); + } + + files = new MojangMappingsDependency(project, getExtension()).resolve(); + } else { + Dependency dependency = project.getDependencies().create(mappings); + files = project.getConfigurations().detachedConfiguration(dependency).resolve(); + } } catch (IllegalDependencyNotation ignored) { project.getLogger().info("Could not locate mappings, presuming V2 Yarn"); diff --git a/src/main/java/net/fabricmc/loom/util/DependencyProvider.java b/src/main/java/net/fabricmc/loom/util/DependencyProvider.java index 0a7ba78..9711669 100644 --- a/src/main/java/net/fabricmc/loom/util/DependencyProvider.java +++ b/src/main/java/net/fabricmc/loom/util/DependencyProvider.java @@ -133,6 +133,10 @@ public abstract class DependencyProvider { } public Set resolve() { + if (dependency instanceof SelfResolvingDependency) { + return ((SelfResolvingDependency) dependency).resolve(); + } + return sourceConfiguration.files(dependency); } diff --git a/src/main/java/net/fabricmc/loom/util/mappings/MojangMappingsDependency.java b/src/main/java/net/fabricmc/loom/util/mappings/MojangMappingsDependency.java new file mode 100644 index 0000000..6be883b --- /dev/null +++ b/src/main/java/net/fabricmc/loom/util/mappings/MojangMappingsDependency.java @@ -0,0 +1,268 @@ +/* + * This file is part of fabric-loom, licensed under the MIT License (MIT). + * + * Copyright (c) 2016, 2017, 2018 FabricMC + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package net.fabricmc.loom.util.mappings; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.StringWriter; +import java.io.Writer; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Set; +import java.util.function.Consumer; + +import org.cadixdev.lorenz.MappingSet; +import org.cadixdev.lorenz.io.TextMappingsWriter; +import org.cadixdev.lorenz.io.proguard.ProGuardReader; +import org.cadixdev.lorenz.model.ClassMapping; +import org.cadixdev.lorenz.model.FieldMapping; +import org.cadixdev.lorenz.model.InnerClassMapping; +import org.cadixdev.lorenz.model.MethodMapping; +import org.cadixdev.lorenz.model.TopLevelClassMapping; +import org.gradle.api.Project; +import org.gradle.api.artifacts.Dependency; +import org.gradle.api.artifacts.SelfResolvingDependency; +import org.gradle.api.tasks.TaskDependency; +import org.zeroturnaround.zip.ByteSource; +import org.zeroturnaround.zip.ZipEntrySource; +import org.zeroturnaround.zip.ZipUtil; + +import net.fabricmc.loom.LoomGradleExtension; +import net.fabricmc.loom.util.DownloadUtil; +import net.fabricmc.loom.util.MinecraftVersionInfo; +import net.fabricmc.lorenztiny.TinyMappingsReader; +import net.fabricmc.mapping.tree.TinyMappingFactory; + +public class MojangMappingsDependency implements SelfResolvingDependency { + private final Project project; + private final LoomGradleExtension extension; + + public MojangMappingsDependency(Project project, LoomGradleExtension extension) { + this.project = project; + this.extension = extension; + } + + @Override + public Set resolve() { + Path mappingsDir = extension.getMappingsProvider().getMappingsDir(); + Path mappingsFile = mappingsDir.resolve(String.format("net.mojang.minecraft-mappings-%s.tiny", getVersion())); + Path clientMappings = mappingsDir.resolve(String.format("net.mojang.minecraft.mappings-%s-client.map", getVersion())); + Path serverMappings = mappingsDir.resolve(String.format("net.mojang.minecraft.mappings-%s-server.map", getVersion())); + + if (!Files.exists(mappingsFile) || project.getGradle().getStartParameter().isRefreshDependencies()) { + MappingSet mappingSet; + + try { + mappingSet = getMappingsSet(clientMappings, serverMappings); + + try (Writer writer = new StringWriter()) { + new TinyWriter(writer, "intermediary", "named").write(mappingSet); + Files.deleteIfExists(mappingsFile); + + ZipUtil.pack(new ZipEntrySource[] { + new ByteSource("mappings/mappings.tiny", writer.toString().getBytes(StandardCharsets.UTF_8)) + }, mappingsFile.toFile()); + } + } catch (IOException e) { + throw new RuntimeException("Failed to resolve Mojang mappings", e); + } + } + + try (BufferedReader clientBufferedReader = Files.newBufferedReader(clientMappings, StandardCharsets.UTF_8)) { + project.getLogger().warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"); + project.getLogger().warn("Using of the official minecraft mappings is at your own risk!"); + project.getLogger().warn("Please make sure to read and understand the following license:"); + project.getLogger().warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"); + String line; + + while ((line = clientBufferedReader.readLine()).startsWith("#")) { + project.getLogger().warn(line); + } + + project.getLogger().warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"); + } catch (IOException e) { + throw new RuntimeException("Failed to read client mappings", e); + } + + return Collections.singleton(mappingsFile.toFile()); + } + + private MappingSet getMappingsSet(Path clientMappings, Path serverMappings) throws IOException { + MinecraftVersionInfo versionInfo = extension.getMinecraftProvider().getVersionInfo(); + + if (versionInfo.downloads.get("client_mappings") == null) { + throw new RuntimeException("Failed to find official mojang mappings for " + getVersion()); + } + + String clientMappingsUrl = versionInfo.downloads.get("client_mappings").url; + String serverMappingsUrl = versionInfo.downloads.get("server_mappings").url; + + DownloadUtil.downloadIfChanged(new URL(clientMappingsUrl), clientMappings.toFile(), project.getLogger()); + DownloadUtil.downloadIfChanged(new URL(serverMappingsUrl), serverMappings.toFile(), project.getLogger()); + + MappingSet mappings = MappingSet.create(); + + try (BufferedReader clientBufferedReader = Files.newBufferedReader(clientMappings, StandardCharsets.UTF_8); + BufferedReader serverBufferedReader = Files.newBufferedReader(serverMappings, StandardCharsets.UTF_8)) { + try (ProGuardReader proGuardReaderClient = new ProGuardReader(clientBufferedReader); + ProGuardReader proGuardReaderServer = new ProGuardReader(serverBufferedReader)) { + proGuardReaderClient.read(mappings); + proGuardReaderServer.read(mappings); + } + } + + MappingSet officialToNamed = mappings.reverse(); + MappingSet intermediaryToOfficial; + + try (BufferedReader reader = Files.newBufferedReader(extension.getMappingsProvider().getIntermediaryTiny(), StandardCharsets.UTF_8)) { + intermediaryToOfficial = new TinyMappingsReader(TinyMappingFactory.loadWithDetection(reader), "intermediary", "official").read(); + } + + MappingSet intermediaryToMojang = MappingSet.create(); + + // Merging. Don't use MappingSet#merge + iterateClasses(intermediaryToOfficial, inputMappings -> { + officialToNamed.getClassMapping(inputMappings.getFullDeobfuscatedName()) + .ifPresent(namedClass -> { + ClassMapping mojangClassMapping = intermediaryToMojang.getOrCreateClassMapping(inputMappings .getFullObfuscatedName()) + .setDeobfuscatedName(namedClass.getFullDeobfuscatedName()); + + for (FieldMapping fieldMapping : inputMappings .getFieldMappings()) { + namedClass.getFieldMapping(fieldMapping.getDeobfuscatedName()) + .ifPresent(namedField -> { + mojangClassMapping.getOrCreateFieldMapping(fieldMapping.getSignature()) + .setDeobfuscatedName(namedField.getDeobfuscatedName()); + }); + } + + for (MethodMapping methodMapping : inputMappings .getMethodMappings()) { + namedClass.getMethodMapping(methodMapping.getDeobfuscatedSignature()) + .ifPresent(namedMethod -> { + mojangClassMapping.getOrCreateMethodMapping(methodMapping.getSignature()) + .setDeobfuscatedName(namedMethod.getDeobfuscatedName()); + }); + } + }); + }); + + return intermediaryToMojang; + } + + @Override + public Set resolve(boolean transitive) { + return resolve(); + } + + @Override + public TaskDependency getBuildDependencies() { + return task -> Collections.emptySet(); + } + + @Override + public String getGroup() { + return "net.mojang.minecraft"; + } + + @Override + public String getName() { + return "mappings"; + } + + @Override + public String getVersion() { + return extension.getMinecraftProvider().getMinecraftVersion(); + } + + @Override + public boolean contentEquals(Dependency dependency) { + if (dependency instanceof MojangMappingsDependency) { + return ((MojangMappingsDependency) dependency).extension.getMinecraftProvider().getMinecraftVersion().equals(getVersion()); + } + + return false; + } + + @Override + public Dependency copy() { + return new MojangMappingsDependency(project, extension); + } + + @Override + public String getReason() { + return null; + } + + @Override + public void because(String s) { + } + + private static void iterateClasses(MappingSet mappings, Consumer> consumer) { + for (TopLevelClassMapping classMapping : mappings.getTopLevelClassMappings()) { + iterateClass(classMapping, consumer); + } + } + + private static void iterateClass(ClassMapping classMapping, Consumer> consumer) { + consumer.accept(classMapping); + + for (InnerClassMapping innerClassMapping : classMapping.getInnerClassMappings()) { + iterateClass(innerClassMapping, consumer); + } + } + + private static class TinyWriter extends TextMappingsWriter { + private final String namespaceFrom; + private final String namespaceTo; + + protected TinyWriter(Writer writer, String namespaceFrom, String namespaceTo) { + super(writer); + this.namespaceFrom = namespaceFrom; + this.namespaceTo = namespaceTo; + } + + @Override + public void write(MappingSet mappings) { + writer.println("tiny\t2\t0\t" + namespaceFrom + "\t" + namespaceTo); + + iterateClasses(mappings, classMapping -> { + writer.println("c\t" + classMapping.getFullObfuscatedName() + "\t" + classMapping.getFullDeobfuscatedName()); + + for (FieldMapping fieldMapping : classMapping.getFieldMappings()) { + fieldMapping.getType().ifPresent(fieldType -> { + writer.println("\tf\t" + fieldType + "\t" + fieldMapping.getObfuscatedName() + "\t" + fieldMapping.getDeobfuscatedName()); + }); + } + + for (MethodMapping methodMapping : classMapping.getMethodMappings()) { + writer.println("\tm\t" + methodMapping.getSignature().getDescriptor() + "\t" + methodMapping.getObfuscatedName() + "\t" + methodMapping.getDeobfuscatedName()); + } + }); + } + } +}