From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Jason Penilla <11360596+jpenilla@users.noreply.github.com> Date: Sun, 20 Jun 2021 18:19:09 -0700 Subject: [PATCH] Deobfuscate stacktraces in log messages, crash reports, and etc. diff --git a/build.gradle.kts b/build.gradle.kts index a482ad74b9a5d06a982ac2a6d9b6c5dc9f664f46..974f0bbbd3d271d28ce884490dc68b68849a07c3 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,7 +1,9 @@ import com.github.jengelman.gradle.plugins.shadow.transformers.Log4j2PluginsCacheFileTransformer import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer +import io.papermc.paperweight.tasks.BaseTask import io.papermc.paperweight.util.* import shadow.org.apache.logging.log4j.core.config.plugins.processor.PluginProcessor.PLUGIN_CACHE_FILE +import java.nio.file.Files import java.util.Locale plugins { @@ -12,6 +14,11 @@ plugins { repositories { maven("https://libraries.minecraft.net/") + // Paper start + maven("https://maven.fabricmc.net/") { + mavenContent { includeModule("net.fabricmc", "mapping-io") } + } + // Paper end } dependencies { @@ -46,6 +53,8 @@ dependencies { implementation("io.netty:netty-all:4.1.65.Final") // Paper + implementation("net.fabricmc:mapping-io:0.3.0") // Paper - needed to read mappings for stacktrace deobfuscation + testImplementation("junit:junit:4.13.1") testImplementation("org.hamcrest:hamcrest-library:1.3") } @@ -95,6 +104,7 @@ relocation { relocate("org.jline:jline-terminal-jansi", "org.jline" to cb("jline")) sequenceOf( + "net.fabricmc:mapping-io" to "net.fabricmc.mappingio", // Paper "commons-codec:commons-codec" to "org.apache.commons.codec", "commons-io:commons-io" to "org.apache.commons.io", //"it.unimi.dsi:fastutil" to "it.unimi", // Paper - don't relocate fastutil @@ -139,6 +149,45 @@ tasks.shadowJar { transform(ModifiedLog4j2PluginsCacheFileTransformer::class.java) } +// Paper start - include reobf mappings in jar for stacktrace deobfuscation +abstract class IncludeMappings : BaseTask() { + @get:InputFile + abstract val inputJar: RegularFileProperty + + @get:InputFile + abstract val mappings: RegularFileProperty + + @get:OutputFile + abstract val outputJar: RegularFileProperty + + override fun init() { + super.init() + outputJar.convention(defaultOutput()) + } + + @TaskAction + private fun addMappings() { + outputJar.get().asFile.parentFile.mkdirs() + inputJar.get().asFile.copyTo(outputJar.get().asFile, overwrite = true) + outputJar.get().path.openZip().use { fs -> + val dir = fs.getPath("META-INF/mappings/") + Files.createDirectories(dir) + val target = dir.resolve("reobf.tiny") + Files.copy(mappings.path, target) + } + } +} + +val includeMappings = tasks.register("includeMappings") { + inputJar.set(tasks.fixJarForReobf.flatMap { it.outputJar }) + mappings.set(tasks.reobfJar.flatMap { it.mappingsFile }) +} + +tasks.reobfJar { + inputJar.set(includeMappings.flatMap { it.outputJar }) +} +// Paper end - include reobf mappings in jar for stacktrace deobfuscation + tasks.test { exclude("org/bukkit/craftbukkit/inventory/ItemStack*Test.class") } diff --git a/src/main/java/com/destroystokyo/paper/PaperConfig.java b/src/main/java/com/destroystokyo/paper/PaperConfig.java index 6f0aba841a3518a36d17ab1e540376384952d2c2..beaa1976b3435176bdfbb9613be80f62bd3f3875 100644 --- a/src/main/java/com/destroystokyo/paper/PaperConfig.java +++ b/src/main/java/com/destroystokyo/paper/PaperConfig.java @@ -437,4 +437,9 @@ public class PaperConfig { log("Async Chunks: Enabled - Chunks will be loaded much faster, without lag."); } } + + public static boolean deobfuscateStacktraces = true; + private static void loggerSettings() { + deobfuscateStacktraces = getBoolean("settings.loggers.deobfuscate-stacktraces", deobfuscateStacktraces); + } } diff --git a/src/main/java/com/destroystokyo/paper/io/SyncLoadFinder.java b/src/main/java/com/destroystokyo/paper/io/SyncLoadFinder.java index 0bb4aaa546939b67a5d22865190f30478a9337c1..d3e619655382e50e9ac9323ed942502d85c9599c 100644 --- a/src/main/java/com/destroystokyo/paper/io/SyncLoadFinder.java +++ b/src/main/java/com/destroystokyo/paper/io/SyncLoadFinder.java @@ -91,7 +91,7 @@ public class SyncLoadFinder { final JsonArray traces = new JsonArray(); - for (StackTraceElement element : pair.getFirst().stacktrace) { + for (StackTraceElement element : io.papermc.paper.util.StacktraceDeobfuscator.INSTANCE.deobfuscateStacktrace(pair.getFirst().stacktrace)) { traces.add(String.valueOf(element)); } diff --git a/src/main/java/io/papermc/paper/logging/StacktraceDeobfuscatingRewritePolicy.java b/src/main/java/io/papermc/paper/logging/StacktraceDeobfuscatingRewritePolicy.java new file mode 100644 index 0000000000000000000000000000000000000000..c701ef3c287f62aa0ebfbdbd6da6ed82a1c7793c --- /dev/null +++ b/src/main/java/io/papermc/paper/logging/StacktraceDeobfuscatingRewritePolicy.java @@ -0,0 +1,38 @@ +package io.papermc.paper.logging; + +import io.papermc.paper.util.StacktraceDeobfuscator; +import org.apache.logging.log4j.core.Core; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.appender.rewrite.RewritePolicy; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.core.impl.Log4jLogEvent; +import org.checkerframework.checker.nullness.qual.NonNull; + +@Plugin( + name = "StacktraceDeobfuscatingRewritePolicy", + category = Core.CATEGORY_NAME, + elementType = "rewritePolicy", + printObject = true +) +public final class StacktraceDeobfuscatingRewritePolicy implements RewritePolicy { + private StacktraceDeobfuscatingRewritePolicy() { + } + + @Override + public @NonNull LogEvent rewrite(final @NonNull LogEvent rewrite) { + final Throwable thrown = rewrite.getThrown(); + if (thrown != null) { + StacktraceDeobfuscator.INSTANCE.deobfuscateThrowable(thrown); + return new Log4jLogEvent.Builder(rewrite) + .setThrownProxy(null) + .build(); + } + return rewrite; + } + + @PluginFactory + public static @NonNull StacktraceDeobfuscatingRewritePolicy createPolicy() { + return new StacktraceDeobfuscatingRewritePolicy(); + } +} diff --git a/src/main/java/io/papermc/paper/util/ObfHelper.java b/src/main/java/io/papermc/paper/util/ObfHelper.java new file mode 100644 index 0000000000000000000000000000000000000000..b8b17d046f836c8652ab094db00ab1af84971b2c --- /dev/null +++ b/src/main/java/io/papermc/paper/util/ObfHelper.java @@ -0,0 +1,146 @@ +package io.papermc.paper.util; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import net.fabricmc.mappingio.MappingReader; +import net.fabricmc.mappingio.format.MappingFormat; +import net.fabricmc.mappingio.tree.MappingTree; +import net.fabricmc.mappingio.tree.MemoryMappingTree; +import org.checkerframework.checker.nullness.qual.NonNull; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.checkerframework.framework.qual.DefaultQualifier; + +@DefaultQualifier(NonNull.class) +public enum ObfHelper { + INSTANCE; + + public static final String MOJANG_PLUS_YARN_NAMESPACE = "mojang+yarn"; + public static final String SPIGOT_NAMESPACE = "spigot"; + + private final @Nullable Map mappingsByObfName; + private final @Nullable Map mappingsByMojangName; + + ObfHelper() { + final @Nullable Set maps = loadMappingsIfPresent(); + if (maps != null) { + this.mappingsByObfName = maps.stream().collect(Collectors.toUnmodifiableMap(ClassMapping::obfName, map -> map)); + this.mappingsByMojangName = maps.stream().collect(Collectors.toUnmodifiableMap(ClassMapping::mojangName, map -> map)); + } else { + this.mappingsByObfName = null; + this.mappingsByMojangName = null; + } + } + + public @Nullable Map mappingsByObfName() { + return this.mappingsByObfName; + } + + public @Nullable Map mappingsByMojangName() { + return this.mappingsByMojangName; + } + + /** + * Attempts to get the obf name for a given class by its Mojang name. Will + * return the input string if mappings are not present. + * + * @param fullyQualifiedMojangName fully qualified class name (dotted) + * @return mapped or original fully qualified (dotted) class name + */ + public String reobfClassName(final String fullyQualifiedMojangName) { + if (this.mappingsByMojangName == null) { + return fullyQualifiedMojangName; + } + + final ClassMapping map = this.mappingsByMojangName.get(fullyQualifiedMojangName); + if (map == null) { + return fullyQualifiedMojangName; + } + + return map.obfName(); + } + + /** + * Attempts to get the Mojang name for a given class by its obf name. Will + * return the input string if mappings are not present. + * + * @param fullyQualifiedObfName fully qualified class name (dotted) + * @return mapped or original fully qualified (dotted) class name + */ + public String deobfClassName(final String fullyQualifiedObfName) { + if (this.mappingsByObfName == null) { + return fullyQualifiedObfName; + } + + final ClassMapping map = this.mappingsByObfName.get(fullyQualifiedObfName); + if (map == null) { + return fullyQualifiedObfName; + } + + return map.mojangName(); + } + + private static @Nullable Set loadMappingsIfPresent() { + try (final @Nullable InputStream mappingsInputStream = ObfHelper.class.getClassLoader().getResourceAsStream("META-INF/mappings/reobf.tiny")) { + if (mappingsInputStream == null) { + return null; + } + final MemoryMappingTree tree = new MemoryMappingTree(); + MappingReader.read(new InputStreamReader(mappingsInputStream, StandardCharsets.UTF_8), MappingFormat.TINY_2, tree); + final Set classes = new HashSet<>(); + + final StringPool pool = new StringPool(); + for (final MappingTree.ClassMapping cls : tree.getClasses()) { + final Map methods = new HashMap<>(); + + for (final MappingTree.MethodMapping methodMapping : cls.getMethods()) { + methods.put( + pool.string(methodKey( + methodMapping.getName(SPIGOT_NAMESPACE), + methodMapping.getDesc(SPIGOT_NAMESPACE) + )), + pool.string(methodMapping.getName(MOJANG_PLUS_YARN_NAMESPACE)) + ); + } + + final ClassMapping map = new ClassMapping( + cls.getName(SPIGOT_NAMESPACE).replace('/', '.'), + cls.getName(MOJANG_PLUS_YARN_NAMESPACE).replace('/', '.'), + Map.copyOf(methods) + ); + classes.add(map); + } + + return Set.copyOf(classes); + } catch (final IOException ex) { + System.err.println("Failed to load mappings for stacktrace deobfuscation."); + ex.printStackTrace(); + return null; + } + } + + public static String methodKey(final String obfName, final String obfDescriptor) { + return obfName + obfDescriptor; + } + + private static final class StringPool { + private final Map pool = new HashMap<>(); + + public String string(final String string) { + return this.pool.computeIfAbsent(string, Function.identity()); + } + } + + public record ClassMapping( + String obfName, + String mojangName, + Map methodsByObf + ) {} +} diff --git a/src/main/java/io/papermc/paper/util/StacktraceDeobfuscator.java b/src/main/java/io/papermc/paper/util/StacktraceDeobfuscator.java new file mode 100644 index 0000000000000000000000000000000000000000..5019d22fa2ab8d72fac5ffcf5dc2a3715ae63760 --- /dev/null +++ b/src/main/java/io/papermc/paper/util/StacktraceDeobfuscator.java @@ -0,0 +1,153 @@ +package io.papermc.paper.util; + +import com.destroystokyo.paper.PaperConfig; +import it.unimi.dsi.fastutil.ints.IntArrayList; +import it.unimi.dsi.fastutil.ints.IntList; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import org.checkerframework.checker.nullness.qual.NonNull; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.checkerframework.framework.qual.DefaultQualifier; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.Label; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; + +@DefaultQualifier(NonNull.class) +public enum StacktraceDeobfuscator { + INSTANCE; + + private final Map, Map> lineMapCache = Collections.synchronizedMap(new LinkedHashMap<>(128, 0.75f, true) { + @Override + protected boolean removeEldestEntry(final Map.Entry, Map> eldest) { + return this.size() > 127; + } + }); + + public void deobfuscateThrowable(final Throwable throwable) { + if (!PaperConfig.deobfuscateStacktraces) { + return; + } + + throwable.setStackTrace(this.deobfuscateStacktrace(throwable.getStackTrace())); + final Throwable cause = throwable.getCause(); + if (cause != null) { + this.deobfuscateThrowable(cause); + } + for (final Throwable suppressed : throwable.getSuppressed()) { + this.deobfuscateThrowable(suppressed); + } + } + + public StackTraceElement[] deobfuscateStacktrace(final StackTraceElement[] traceElements) { + if (!PaperConfig.deobfuscateStacktraces) { + return traceElements; + } + + final @Nullable Map mappings = ObfHelper.INSTANCE.mappingsByObfName(); + if (mappings == null || traceElements.length == 0) { + return traceElements; + } + final StackTraceElement[] result = new StackTraceElement[traceElements.length]; + for (int i = 0; i < traceElements.length; i++) { + final StackTraceElement element = traceElements[i]; + + final String className = element.getClassName(); + final String methodName = element.getMethodName(); + + final ObfHelper.ClassMapping classMapping = mappings.get(className); + if (classMapping == null) { + result[i] = element; + continue; + } + + final Class clazz; + try { + clazz = Class.forName(className); + } catch (final ClassNotFoundException ex) { + throw new RuntimeException(ex); + } + final @Nullable String methodKey = this.determineMethodForLine(clazz, element.getLineNumber()); + final @Nullable String mappedMethodName = methodKey == null ? null : classMapping.methodsByObf().get(methodKey); + + result[i] = new StackTraceElement( + element.getClassLoaderName(), + element.getModuleName(), + element.getModuleVersion(), + classMapping.mojangName(), + mappedMethodName != null ? mappedMethodName : methodName, + sourceFileName(classMapping.mojangName()), + element.getLineNumber() + ); + } + return result; + } + + private @Nullable String determineMethodForLine(final Class clazz, final int lineNumber) { + final Map lineMap = this.lineMapCache.computeIfAbsent(clazz, StacktraceDeobfuscator::buildLineMap); + for (final var entry : lineMap.entrySet()) { + final String methodKey = entry.getKey(); + final IntList lines = entry.getValue(); + for (int i = 0, linesSize = lines.size(); i < linesSize; i++) { + final int num = lines.getInt(i); + if (num == lineNumber) { + return methodKey; + } + } + } + return null; + } + + private static String sourceFileName(final String fullClassName) { + final int dot = fullClassName.lastIndexOf('.'); + final String className = dot == -1 + ? fullClassName + : fullClassName.substring(dot + 1); + final String rootClassName = className.split("\\$")[0]; + return rootClassName + ".java"; + } + + private static Map buildLineMap(final Class key) { + final Map lineMap = new HashMap<>(); + final class LineCollectingMethodVisitor extends MethodVisitor { + private final IntList lines = new IntArrayList(); + private final String name; + private final String descriptor; + + LineCollectingMethodVisitor(String name, String descriptor) { + super(Opcodes.ASM9); + this.name = name; + this.descriptor = descriptor; + } + + @Override + public void visitLineNumber(int line, Label start) { + super.visitLineNumber(line, start); + this.lines.add(line); + } + + @Override + public void visitEnd() { + super.visitEnd(); + lineMap.put(ObfHelper.methodKey(this.name, this.descriptor), this.lines); + } + } + final ClassVisitor classVisitor = new ClassVisitor(Opcodes.ASM9) { + @Override + public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { + return new LineCollectingMethodVisitor(name, descriptor); + } + }; + try { + final ClassReader reader = new ClassReader(key.getName()); + reader.accept(classVisitor, 0); + } catch (final IOException ex) { + throw new RuntimeException(ex); + } + return lineMap; + } +} diff --git a/src/main/java/io/papermc/paper/util/TraceUtil.java b/src/main/java/io/papermc/paper/util/TraceUtil.java index 2d5494d2813b773e60ddba6790b750a9a08f21f8..7695bf44503f161523ea612ef8a884ae574a2e21 100644 --- a/src/main/java/io/papermc/paper/util/TraceUtil.java +++ b/src/main/java/io/papermc/paper/util/TraceUtil.java @@ -6,13 +6,15 @@ public final class TraceUtil { public static void dumpTraceForThread(Thread thread, String reason) { Bukkit.getLogger().warning(thread.getName() + ": " + reason); - StackTraceElement[] trace = thread.getStackTrace(); + StackTraceElement[] trace = StacktraceDeobfuscator.INSTANCE.deobfuscateStacktrace(thread.getStackTrace()); for (StackTraceElement traceElement : trace) { Bukkit.getLogger().warning("\tat " + traceElement); } } public static void dumpTraceForThread(String reason) { - new Throwable(reason).printStackTrace(); + final Throwable throwable = new Throwable(reason); + StacktraceDeobfuscator.INSTANCE.deobfuscateThrowable(throwable); + throwable.printStackTrace(); } } diff --git a/src/main/java/net/minecraft/CrashReport.java b/src/main/java/net/minecraft/CrashReport.java index 62c3c597732e6fb30ed5367d902ea8763507a6b8..b9a618eba86656289e20d051c3f0023ec93bb733 100644 --- a/src/main/java/net/minecraft/CrashReport.java +++ b/src/main/java/net/minecraft/CrashReport.java @@ -30,6 +30,7 @@ public class CrashReport { private final SystemReport systemReport = new SystemReport(); public CrashReport(String message, Throwable cause) { + io.papermc.paper.util.StacktraceDeobfuscator.INSTANCE.deobfuscateThrowable(cause); // Paper this.title = message; this.exception = cause; this.systemReport.setDetail("CraftBukkit Information", new org.bukkit.craftbukkit.CraftCrashReport()); // CraftBukkit diff --git a/src/main/java/net/minecraft/CrashReportCategory.java b/src/main/java/net/minecraft/CrashReportCategory.java index 3941e14d1c3e6e688e28904948039c8b2200de5f..a4fda4a3bae9ce600e778b44cd3ef432a8b65667 100644 --- a/src/main/java/net/minecraft/CrashReportCategory.java +++ b/src/main/java/net/minecraft/CrashReportCategory.java @@ -104,6 +104,7 @@ public class CrashReportCategory { } else { this.stackTrace = new StackTraceElement[stackTraceElements.length - 3 - ignoredCallCount]; System.arraycopy(stackTraceElements, 3 + ignoredCallCount, this.stackTrace, 0, this.stackTrace.length); + this.stackTrace = io.papermc.paper.util.StacktraceDeobfuscator.INSTANCE.deobfuscateStacktrace(this.stackTrace); // Paper return this.stackTrace.length; } } diff --git a/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java b/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java index a335d48467d1730bfed25eb5fd9046e115f23ed0..8098875a5c4146dec81f5daed0e34ddfe17a26c6 100644 --- a/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java +++ b/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java @@ -214,6 +214,7 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface } com.destroystokyo.paper.PaperConfig.registerCommands(); com.destroystokyo.paper.VersionHistoryManager.INSTANCE.getClass(); // load version history now + io.papermc.paper.util.ObfHelper.INSTANCE.getClass(); // load mappings for stacktrace deobf and etc. // Paper end this.setPvpAllowed(dedicatedserverproperties.pvp); diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java index 4d271cae88c16ed2419f896c728fdff612540500..dcfbe77bdb25d9c58ffb7b75c48bdb580bc0de47 100644 --- a/src/main/java/org/spigotmc/WatchdogThread.java +++ b/src/main/java/org/spigotmc/WatchdogThread.java @@ -106,7 +106,7 @@ public class WatchdogThread extends Thread log.log( Level.SEVERE, "During the run of the server, a plugin set an excessive velocity on an entity" ); log.log( Level.SEVERE, "This may be the cause of the issue, or it may be entirely unrelated" ); log.log( Level.SEVERE, org.bukkit.craftbukkit.CraftServer.excessiveVelEx.getMessage()); - for ( StackTraceElement stack : org.bukkit.craftbukkit.CraftServer.excessiveVelEx.getStackTrace() ) + for ( StackTraceElement stack : io.papermc.paper.util.StacktraceDeobfuscator.INSTANCE.deobfuscateStacktrace(org.bukkit.craftbukkit.CraftServer.excessiveVelEx.getStackTrace()) ) // Paper { log.log( Level.SEVERE, "\t\t" + stack ); } @@ -194,7 +194,7 @@ public class WatchdogThread extends Thread } log.log( Level.SEVERE, "\tStack:" ); // - for ( StackTraceElement stack : thread.getStackTrace() ) + for ( StackTraceElement stack : io.papermc.paper.util.StacktraceDeobfuscator.INSTANCE.deobfuscateStacktrace(thread.getStackTrace()) ) // Paper { log.log( Level.SEVERE, "\t\t" + stack ); } diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml index d285dbec16272db6b8a71865e05924ad66087407..1a05d23ff886b015fb9396f119822c678a47ec6f 100644 --- a/src/main/resources/log4j2.xml +++ b/src/main/resources/log4j2.xml @@ -30,10 +30,14 @@ + + + + - +