From ce52582f49e618729720a057ae5029d2a1d15da4 Mon Sep 17 00:00:00 2001 From: liach Date: Wed, 4 Sep 2019 11:56:09 -0400 Subject: Add tinyv2 save/load --- .../cuchaz/enigma/command/MappingCommandsUtil.java | 19 +- .../translation/mapping/serde/MappingFormat.java | 1 + .../translation/mapping/serde/RawEntryMapping.java | 23 ++ .../translation/mapping/serde/TinyV2Reader.java | 290 +++++++++++++++++++++ .../translation/mapping/serde/TinyV2Writer.java | 169 ++++++++++++ 5 files changed, 500 insertions(+), 2 deletions(-) create mode 100644 src/main/java/cuchaz/enigma/translation/mapping/serde/RawEntryMapping.java create mode 100644 src/main/java/cuchaz/enigma/translation/mapping/serde/TinyV2Reader.java create mode 100644 src/main/java/cuchaz/enigma/translation/mapping/serde/TinyV2Writer.java (limited to 'src/main/java/cuchaz/enigma') diff --git a/src/main/java/cuchaz/enigma/command/MappingCommandsUtil.java b/src/main/java/cuchaz/enigma/command/MappingCommandsUtil.java index bacb8ff..fc68edf 100644 --- a/src/main/java/cuchaz/enigma/command/MappingCommandsUtil.java +++ b/src/main/java/cuchaz/enigma/command/MappingCommandsUtil.java @@ -91,7 +91,11 @@ public final class MappingCommandsUtil { MappingFormat format = null; try { format = MappingFormat.valueOf(type.toUpperCase()); - } catch (IllegalArgumentException ignored) {} + } catch (IllegalArgumentException ignored) { + if (type.equals("tinyv2")) { + format = MappingFormat.TINY_V2; + } + } if (format != null) { return format.getReader().read(path, ProgressListener.none(), saveParameters); @@ -106,7 +110,18 @@ public final class MappingCommandsUtil { return; } - if (type.startsWith("tiny")) { + if (type.startsWith("tinyv2:") || type.startsWith("tiny_v2:")) { + String[] split = type.split(":"); + + if (split.length != 3) { + throw new IllegalArgumentException("specify column names as 'tinyv2:from_namespace:to_namespace'"); + } + + new TinyV2Writer(split[1], split[2]).write(mappings, path, ProgressListener.none(), saveParameters); + return; + } + + if (type.startsWith("tiny:")) { String[] split = type.split(":"); if (split.length != 3) { diff --git a/src/main/java/cuchaz/enigma/translation/mapping/serde/MappingFormat.java b/src/main/java/cuchaz/enigma/translation/mapping/serde/MappingFormat.java index e6461b4..7eae1c0 100644 --- a/src/main/java/cuchaz/enigma/translation/mapping/serde/MappingFormat.java +++ b/src/main/java/cuchaz/enigma/translation/mapping/serde/MappingFormat.java @@ -14,6 +14,7 @@ import java.nio.file.Path; public enum MappingFormat { ENIGMA_FILE(EnigmaMappingsWriter.FILE, EnigmaMappingsReader.FILE), ENIGMA_DIRECTORY(EnigmaMappingsWriter.DIRECTORY, EnigmaMappingsReader.DIRECTORY), + TINY_V2(new TinyV2Writer("intermediary", "named"), new TinyV2Reader()), TINY_FILE(TinyMappingsWriter.INSTANCE, TinyMappingsReader.INSTANCE), SRG_FILE(SrgMappingsWriter.INSTANCE, null), PROGUARD(null, ProguardMappingsReader.INSTANCE); diff --git a/src/main/java/cuchaz/enigma/translation/mapping/serde/RawEntryMapping.java b/src/main/java/cuchaz/enigma/translation/mapping/serde/RawEntryMapping.java new file mode 100644 index 0000000..5d63f38 --- /dev/null +++ b/src/main/java/cuchaz/enigma/translation/mapping/serde/RawEntryMapping.java @@ -0,0 +1,23 @@ +package cuchaz.enigma.translation.mapping.serde; + +import com.google.common.base.Strings; +import cuchaz.enigma.translation.mapping.AccessModifier; +import cuchaz.enigma.translation.mapping.EntryMapping; + +final class RawEntryMapping { + private final String targetName; + private final AccessModifier access; + + RawEntryMapping(String targetName) { + this(targetName, null); + } + + RawEntryMapping(String targetName, AccessModifier access) { + this.access = access; + this.targetName = targetName; + } + + EntryMapping bake() { + return Strings.isNullOrEmpty(targetName) ? null : new EntryMapping(targetName, access); + } +} diff --git a/src/main/java/cuchaz/enigma/translation/mapping/serde/TinyV2Reader.java b/src/main/java/cuchaz/enigma/translation/mapping/serde/TinyV2Reader.java new file mode 100644 index 0000000..2621f31 --- /dev/null +++ b/src/main/java/cuchaz/enigma/translation/mapping/serde/TinyV2Reader.java @@ -0,0 +1,290 @@ +package cuchaz.enigma.translation.mapping.serde; + +import cuchaz.enigma.ProgressListener; +import cuchaz.enigma.throwables.MappingParseException; +import cuchaz.enigma.translation.mapping.EntryMapping; +import cuchaz.enigma.translation.mapping.MappingPair; +import cuchaz.enigma.translation.mapping.MappingSaveParameters; +import cuchaz.enigma.translation.mapping.tree.EntryTree; +import cuchaz.enigma.translation.mapping.tree.HashEntryTree; +import cuchaz.enigma.translation.representation.MethodDescriptor; +import cuchaz.enigma.translation.representation.TypeDescriptor; +import cuchaz.enigma.translation.representation.entry.ClassEntry; +import cuchaz.enigma.translation.representation.entry.Entry; +import cuchaz.enigma.translation.representation.entry.FieldEntry; +import cuchaz.enigma.translation.representation.entry.LocalVariableEntry; +import cuchaz.enigma.translation.representation.entry.MethodEntry; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.BitSet; +import java.util.List; + +final class TinyV2Reader implements MappingsReader { + + private static final String MINOR_VERSION = "0"; + // 0 indent + private static final int IN_HEADER = 0; + private static final int IN_CLASS = IN_HEADER + 1; + // 1 indent + private static final int IN_METHOD = IN_CLASS + 1; + private static final int IN_FIELD = IN_METHOD + 1; + // 2 indent + private static final int IN_PARAMETER = IN_FIELD + 1; + // general properties + private static final int STATE_SIZE = IN_PARAMETER + 1; + private static final int[] INDENT_CLEAR_START = {IN_HEADER, IN_METHOD, IN_PARAMETER, STATE_SIZE}; + + @Override + public EntryTree read(Path path, ProgressListener progress, MappingSaveParameters saveParameters) throws IOException, MappingParseException { + return read(path, Files.readAllLines(path, StandardCharsets.UTF_8), progress); + } + + private EntryTree read(Path path, List lines, ProgressListener progress) throws MappingParseException { + EntryTree mappings = new HashEntryTree<>(); + + progress.init(lines.size(), "Loading mapping file"); + + BitSet state = new BitSet(STATE_SIZE); + @SuppressWarnings({"unchecked", "rawtypes"}) + MappingPair, RawEntryMapping>[] holds = new MappingPair[STATE_SIZE]; + boolean escapeNames = false; + + for (int lineNumber = 0; lineNumber < lines.size(); lineNumber++) { + try { + progress.step(lineNumber, ""); + String line = lines.get(lineNumber); + + int indent = 0; + while (line.charAt(indent) == '\t') + indent++; + + String[] parts = line.substring(indent).split("\t", -1); + if (parts.length == 0 || indent >= INDENT_CLEAR_START.length) + throw new IllegalArgumentException("Invalid format"); + + // clean and register stuff in stack + for (int i = INDENT_CLEAR_START[indent]; i < STATE_SIZE; i++) { + state.clear(i); + if (holds[i] != null) { + RawEntryMapping mapping = holds[i].getMapping(); + if (mapping != null) { + EntryMapping baked = mapping.bake(); + if (baked != null) { + mappings.insert(holds[i].getEntry(), baked); + } + } + holds[i] = null; + } + } + + switch (indent) { + case 0: + switch (parts[0]) { + case "tiny": // header + if (lineNumber != 0) { + throw new IllegalArgumentException("Header can only be on the first line"); + } + if (parts.length < 5) { + throw new IllegalArgumentException("Not enough header columns, needs at least 5"); + } + if (!"2".equals(parts[1]) || !MINOR_VERSION.equals(parts[2])) { + throw new IllegalArgumentException("Unsupported TinyV2 version, requires major " + "2" + " and minor " + MINOR_VERSION + ""); + } + state.set(IN_HEADER); + break; + case "c": // class + state.set(IN_CLASS); + holds[IN_CLASS] = parseClass(parts, escapeNames); + break; + default: + unsupportKey(parts); + } + + break; + case 1: + if (state.get(IN_HEADER)) { + if (parts[0].equals("esacpe-names")) { + escapeNames = true; + } + + break; + } + + if (state.get(IN_CLASS)) { + switch (parts[0]) { + case "m": // method + state.set(IN_METHOD); + holds[IN_METHOD] = parseMethod(holds[IN_CLASS], parts, escapeNames); + break; + case "f": // field + state.set(IN_FIELD); + holds[IN_FIELD] = parseField(holds[IN_CLASS], parts, escapeNames); + break; + case "c": // class javadoc + addJavadoc(holds[IN_CLASS], parts); + break; + default: + unsupportKey(parts); + } + break; + } + + unsupportKey(parts); + case 2: + if (state.get(IN_METHOD)) { + switch (parts[0]) { + case "p": // parameter + state.set(IN_PARAMETER); + holds[IN_PARAMETER] = parseArgument(holds[IN_METHOD], parts, escapeNames); + break; + case "v": // local variable + // TODO add local var mapping + break; + case "c": // method javadoc + addJavadoc(holds[IN_METHOD], parts); + break; + default: + unsupportKey(parts); + } + break; + } + + if (state.get(IN_FIELD)) { + switch (parts[0]) { + case "c": // field javadoc + addJavadoc(holds[IN_FIELD], parts); + break; + default: + unsupportKey(parts); + } + break; + } + unsupportKey(parts); + case 3: + if (state.get(IN_PARAMETER)) { + switch (parts[0]) { + case "c": + addJavadoc(holds[IN_PARAMETER], parts); + break; + default: + unsupportKey(parts); + } + break; + } + unsupportKey(parts); + default: + unsupportKey(parts); + } + + } catch (Throwable t) { + t.printStackTrace(); + throw new MappingParseException(path::toString, lineNumber + 1, t.toString()); + } + } + + return mappings; + } + + private void unsupportKey(String[] parts) { + throw new IllegalArgumentException("Unsupported key " + parts[0]); + } + + private void addJavadoc(MappingPair pair, String[] parts) { + if (parts.length != 2) { + throw new IllegalArgumentException("Invalid javadoc declaration"); + } + + addJavadoc(pair, parts[1]); + } + + private void addJavadoc(MappingPair pair, String javadoc) { + RawEntryMapping mapping = pair.getMapping(); + if (mapping == null) { + throw new IllegalArgumentException("Javadoc requires a mapping in enigma!"); + } +// mapping.addJavadocLine(javadoc); todo javadocs + } + + private MappingPair parseClass(String[] tokens, boolean escapeNames) { + ClassEntry obfuscatedEntry = new ClassEntry(unescapeOpt(tokens[1], escapeNames)); + if (tokens.length <= 2) + return new MappingPair<>(obfuscatedEntry); + String token2 = unescapeOpt(tokens[2], escapeNames); + String mapping = token2.substring(token2.lastIndexOf('$') + 1); + return new MappingPair<>(obfuscatedEntry, new RawEntryMapping(mapping)); + } + + private MappingPair parseField(MappingPair parent, String[] tokens, boolean escapeNames) { + ClassEntry ownerClass = (ClassEntry) parent.getEntry(); + TypeDescriptor descriptor = new TypeDescriptor(unescapeOpt(tokens[1], escapeNames)); + + FieldEntry obfuscatedEntry = new FieldEntry(ownerClass, unescapeOpt(tokens[2], escapeNames), descriptor); + if (tokens.length <= 3) + return new MappingPair<>(obfuscatedEntry); + String mapping = unescapeOpt(tokens[3], escapeNames); + return new MappingPair<>(obfuscatedEntry, new RawEntryMapping(mapping)); + } + + private MappingPair parseMethod(MappingPair parent, String[] tokens, boolean escapeNames) { + ClassEntry ownerClass = (ClassEntry) parent.getEntry(); + MethodDescriptor descriptor = new MethodDescriptor(unescapeOpt(tokens[1], escapeNames)); + + MethodEntry obfuscatedEntry = new MethodEntry(ownerClass, unescapeOpt(tokens[2], escapeNames), descriptor); + if (tokens.length <= 3) + return new MappingPair<>(obfuscatedEntry); + String mapping = unescapeOpt(tokens[3], escapeNames); + return new MappingPair<>(obfuscatedEntry, new RawEntryMapping(mapping)); + } + + private MappingPair parseArgument(MappingPair parent, String[] tokens, boolean escapeNames) { + MethodEntry ownerMethod = (MethodEntry) parent.getEntry(); + int variableIndex = Integer.parseInt(tokens[1]); + + // tokens[2] is the useless obf name + + LocalVariableEntry obfuscatedEntry = new LocalVariableEntry(ownerMethod, variableIndex, "", true); + if (tokens.length <= 3) + return new MappingPair<>(obfuscatedEntry); + String mapping = unescapeOpt(tokens[3], escapeNames); + return new MappingPair<>(obfuscatedEntry, new RawEntryMapping(mapping)); + } + + private static final String TO_ESCAPE = "\\\n\r\0\t"; + private static final String ESCAPED = "\\nr0t"; + + private static String unescapeOpt(String raw, boolean escapedStrings) { + return escapedStrings ? unescape(raw) : raw; + } + + private static String unescape(String str) { + // copied from matcher, lazy! + int pos = str.indexOf('\\'); + if (pos < 0) return str; + + StringBuilder ret = new StringBuilder(str.length() - 1); + int start = 0; + + do { + ret.append(str, start, pos); + pos++; + int type; + + if (pos >= str.length()) { + throw new RuntimeException("incomplete escape sequence at the end"); + } else if ((type = ESCAPED.indexOf(str.charAt(pos))) < 0) { + throw new RuntimeException("invalid escape character: \\" + str.charAt(pos)); + } else { + ret.append(TO_ESCAPE.charAt(type)); + } + + start = pos + 1; + } while ((pos = str.indexOf('\\', start)) >= 0); + + ret.append(str, start, str.length()); + + return ret.toString(); + } +} diff --git a/src/main/java/cuchaz/enigma/translation/mapping/serde/TinyV2Writer.java b/src/main/java/cuchaz/enigma/translation/mapping/serde/TinyV2Writer.java new file mode 100644 index 0000000..a734ca2 --- /dev/null +++ b/src/main/java/cuchaz/enigma/translation/mapping/serde/TinyV2Writer.java @@ -0,0 +1,169 @@ +package cuchaz.enigma.translation.mapping.serde; + +import com.google.common.base.Strings; +import cuchaz.enigma.ProgressListener; +import cuchaz.enigma.translation.mapping.EntryMap; +import cuchaz.enigma.translation.mapping.EntryMapping; +import cuchaz.enigma.translation.mapping.MappingDelta; +import cuchaz.enigma.translation.mapping.MappingSaveParameters; +import cuchaz.enigma.translation.mapping.tree.EntryTree; +import cuchaz.enigma.translation.mapping.tree.EntryTreeNode; +import cuchaz.enigma.translation.representation.entry.ClassEntry; +import cuchaz.enigma.translation.representation.entry.Entry; +import cuchaz.enigma.translation.representation.entry.FieldEntry; +import cuchaz.enigma.translation.representation.entry.LocalVariableEntry; +import cuchaz.enigma.translation.representation.entry.MethodEntry; +import cuchaz.enigma.utils.LFPrintWriter; + +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Deque; +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +public final class TinyV2Writer implements MappingsWriter { + + private static final String MINOR_VERSION = "0"; + private final String obfHeader; + private final String deobfHeader; + + public TinyV2Writer(String obfHeader, String deobfHeader) { + this.obfHeader = obfHeader; + this.deobfHeader = deobfHeader; + } + + @Override + public void write(EntryTree mappings, MappingDelta delta, Path path, ProgressListener progress, MappingSaveParameters parameters) { + List> classes = StreamSupport.stream(mappings.spliterator(), false).filter(node -> node.getEntry() instanceof ClassEntry).collect(Collectors.toList()); + + try (PrintWriter writer = new LFPrintWriter(Files.newBufferedWriter(path))) { + writer.println("tiny\t2\t" + MINOR_VERSION + "\t" + obfHeader + "\t" + deobfHeader); + + // no escape names + + for (EntryTreeNode node : classes) { + writeClass(writer, node, mappings); + } + } catch (IOException ex) { + ex.printStackTrace(); // TODO add some better logging system + } + } + + private void writeClass(PrintWriter writer, EntryTreeNode node, EntryMap tree) { + writer.print("c\t"); + ClassEntry classEntry = (ClassEntry) node.getEntry(); + String fullName = classEntry.getFullName(); + writer.print(fullName); + Deque parts = new LinkedList<>(); + do { + EntryMapping mapping = tree.get(classEntry); + if (mapping != null) { + parts.addFirst(mapping.getTargetName()); + } else { + parts.addFirst(classEntry.getName()); + } + classEntry = classEntry.getOuterClass(); + } while (classEntry != null); + + String mappedName = String.join("$", parts); + + writer.print("\t"); + + writer.print(mappedName); // todo escaping when we have v2 fixed later + + writer.println(); + + writeComment(writer, node.getValue(), 1); + + for (EntryTreeNode child : node.getChildNodes()) { + Entry entry = child.getEntry(); + if (entry instanceof FieldEntry) { + writeField(writer, child); + } else if (entry instanceof MethodEntry) { + writeMethod(writer, child); + } + } + } + + private void writeMethod(PrintWriter writer, EntryTreeNode node) { + writer.print(indent(1)); + writer.print("m\t"); + writer.print(((MethodEntry) node.getEntry()).getDesc().toString()); + writer.print("\t"); + writer.print(node.getEntry().getName()); + writer.print("\t"); + EntryMapping mapping = node.getValue(); + if (mapping == null) { + writer.println(node.getEntry().getName()); // todo fix v2 name inference + } else { + writer.println(mapping.getTargetName()); + + writeComment(writer, mapping, 2); + } + + for (EntryTreeNode child : node.getChildNodes()) { + Entry entry = child.getEntry(); + if (entry instanceof LocalVariableEntry) { + writeParameter(writer, child); + } + // TODO write actual local variables + } + } + + private void writeField(PrintWriter writer, EntryTreeNode node) { + if (node.getValue() == null) + return; // Shortcut + + writer.print(indent(1)); + writer.print("f\t"); + writer.print(((FieldEntry) node.getEntry()).getDesc().toString()); + writer.print("\t"); + writer.print(node.getEntry().getName()); + writer.print("\t"); + EntryMapping mapping = node.getValue(); + if (mapping == null) { + writer.println(node.getEntry().getName()); // todo fix v2 name inference + } else { + writer.println(mapping.getTargetName()); + + writeComment(writer, mapping, 2); + } + } + + private void writeParameter(PrintWriter writer, EntryTreeNode node) { + if (node.getValue() == null) + return; // Shortcut + + writer.print(indent(2)); + writer.print("p\t"); + writer.print(((LocalVariableEntry) node.getEntry()).getIndex()); + writer.print("\t"); + writer.print(node.getEntry().getName()); + writer.print("\t"); + EntryMapping mapping = node.getValue(); + if (mapping == null) { + writer.println(); // todo ??? + } else { + writer.println(mapping.getTargetName()); + + writeComment(writer, mapping, 3); + } + } + + private void writeComment(PrintWriter writer, EntryMapping mapping, int indent) { +// if (mapping != null && mapping.getJavadoc() != null) { todo javadocs +// writer.print(indent(indent)); +// writer.print("c\t"); +// writer.print(MappingHelper.escape(mapping.getJavadoc())); +// writer.println(); +// } + } + + private String indent(int level) { + return Strings.repeat("\t", level); + } +} -- cgit v1.2.3