/******************************************************************************* * Copyright (c) 2015 Jeff Martin. * All rights reserved. This program and the accompanying materials * are made available under the terms of the GNU Lesser General Public * License v3.0 which accompanies this distribution, and is available at * http://www.gnu.org/licenses/lgpl.html *

* Contributors: * Jeff Martin - initial API and implementation ******************************************************************************/ package cuchaz.enigma.convert; import com.google.common.collect.*; import cuchaz.enigma.Deobfuscator; import cuchaz.enigma.TranslatingTypeLoader; import cuchaz.enigma.analysis.JarIndex; import cuchaz.enigma.convert.ClassNamer.SidedClassNamer; import cuchaz.enigma.mapping.*; import cuchaz.enigma.throwables.MappingConflict; import javassist.CtClass; import javassist.CtMethod; import javassist.NotFoundException; import javassist.bytecode.BadBytecode; import javassist.bytecode.CodeAttribute; import javassist.bytecode.CodeIterator; import java.util.*; import java.util.jar.JarFile; public class MappingsConverter { public static ClassMatches computeClassMatches(JarFile sourceJar, JarFile destJar, Mappings mappings) { // index jars System.out.println("Indexing source jar..."); JarIndex sourceIndex = new JarIndex(); sourceIndex.indexJar(sourceJar, false); System.out.println("Indexing dest jar..."); JarIndex destIndex = new JarIndex(); destIndex.indexJar(destJar, false); // compute the matching ClassMatching matching = computeMatching(sourceJar, sourceIndex, destJar, destIndex, null); return new ClassMatches(matching.matches()); } public static ClassMatching computeMatching(JarFile sourceJar, JarIndex sourceIndex, JarFile destJar, JarIndex destIndex, BiMap knownMatches) { System.out.println("Iteratively matching classes"); ClassMatching lastMatching = null; int round = 0; SidedClassNamer sourceNamer = null; SidedClassNamer destNamer = null; for (boolean useReferences : Arrays.asList(false, true)) { int numUniqueMatchesLastTime = 0; if (lastMatching != null) { numUniqueMatchesLastTime = lastMatching.uniqueMatches().size(); } while (true) { System.out.println("Round " + (++round) + "..."); // init the matching with identity settings ClassMatching matching = new ClassMatching( new ClassIdentifier(sourceJar, sourceIndex, sourceNamer, useReferences), new ClassIdentifier(destJar, destIndex, destNamer, useReferences) ); if (knownMatches != null) { matching.addKnownMatches(knownMatches); } if (lastMatching == null) { // search all classes matching.match(sourceIndex.getObfClassEntries(), destIndex.getObfClassEntries()); } else { // we already know about these matches from last time matching.addKnownMatches(lastMatching.uniqueMatches()); // search unmatched and ambiguously-matched classes matching.match(lastMatching.unmatchedSourceClasses(), lastMatching.unmatchedDestClasses()); for (ClassMatch match : lastMatching.ambiguousMatches()) { matching.match(match.sourceClasses, match.destClasses); } } System.out.println(matching); BiMap uniqueMatches = matching.uniqueMatches(); // did we match anything new this time? if (uniqueMatches.size() > numUniqueMatchesLastTime) { numUniqueMatchesLastTime = uniqueMatches.size(); lastMatching = matching; } else { break; } // update the namers ClassNamer namer = new ClassNamer(uniqueMatches); sourceNamer = namer.getSourceNamer(); destNamer = namer.getDestNamer(); } } return lastMatching; } public static Mappings newMappings(ClassMatches matches, Mappings oldMappings, Deobfuscator sourceDeobfuscator, Deobfuscator destDeobfuscator) throws MappingConflict { // sort the unique matches by size of inner class chain Multimap> matchesByDestChainSize = HashMultimap.create(); for (java.util.Map.Entry match : matches.getUniqueMatches().entrySet()) { int chainSize = destDeobfuscator.getJarIndex().getObfClassChain(match.getValue()).size(); matchesByDestChainSize.put(chainSize, match); } // build the mappings (in order of small-to-large inner chains) Mappings newMappings = new Mappings(); List chainSizes = Lists.newArrayList(matchesByDestChainSize.keySet()); Collections.sort(chainSizes); for (int chainSize : chainSizes) { for (java.util.Map.Entry match : matchesByDestChainSize.get(chainSize)) { // get class info ClassEntry obfSourceClassEntry = match.getKey(); ClassEntry obfDestClassEntry = match.getValue(); List destClassChain = destDeobfuscator.getJarIndex().getObfClassChain(obfDestClassEntry); ClassMapping sourceMapping; if (obfSourceClassEntry.isInnerClass()) { List srcClassChain = sourceDeobfuscator.getMappings().getClassMappingChain(obfSourceClassEntry); sourceMapping = srcClassChain.get(srcClassChain.size() - 1); } else { sourceMapping = sourceDeobfuscator.getMappings().getClassByObf(obfSourceClassEntry); } if (sourceMapping == null) { // if this class was never deobfuscated, don't try to match it continue; } // find out where to make the dest class mapping if (destClassChain.size() == 1) { // not an inner class, add directly to mappings newMappings.addClassMapping(migrateClassMapping(obfDestClassEntry, sourceMapping, matches, false)); } else { // inner class, find the outer class mapping ClassMapping destMapping = null; for (int i = 0; i < destClassChain.size() - 1; i++) { ClassEntry destChainClassEntry = destClassChain.get(i); if (destMapping == null) { destMapping = newMappings.getClassByObf(destChainClassEntry); if (destMapping == null) { destMapping = new ClassMapping(destChainClassEntry.getName()); newMappings.addClassMapping(destMapping); } } else { destMapping = destMapping.getInnerClassByObfSimple(destChainClassEntry.getInnermostClassName()); if (destMapping == null) { destMapping = new ClassMapping(destChainClassEntry.getName()); destMapping.addInnerClassMapping(destMapping); } } } destMapping.addInnerClassMapping(migrateClassMapping(obfDestClassEntry, sourceMapping, matches, true)); } } } return newMappings; } private static ClassMapping migrateClassMapping(ClassEntry newObfClass, ClassMapping oldClassMapping, final ClassMatches matches, boolean useSimpleName) { ClassNameReplacer replacer = className -> { ClassEntry newClassEntry = matches.getUniqueMatches().get(new ClassEntry(className)); if (newClassEntry != null) { return newClassEntry.getName(); } return null; }; ClassMapping newClassMapping; String deobfName = oldClassMapping.getDeobfName(); if (deobfName != null) { if (useSimpleName) { deobfName = new ClassEntry(deobfName).getSimpleName(); } newClassMapping = new ClassMapping(newObfClass.getName(), deobfName); } else { newClassMapping = new ClassMapping(newObfClass.getName()); } // migrate fields for (FieldMapping oldFieldMapping : oldClassMapping.fields()) { if (canMigrate(oldFieldMapping.getObfType(), matches)) { newClassMapping.addFieldMapping(new FieldMapping(oldFieldMapping, replacer)); } else { System.out.println(String.format("Can't map field, dropping: %s.%s %s", oldClassMapping.getDeobfName(), oldFieldMapping.getDeobfName(), oldFieldMapping.getObfType() )); } } // migrate methods for (MethodMapping oldMethodMapping : oldClassMapping.methods()) { if (canMigrate(oldMethodMapping.getObfSignature(), matches)) { newClassMapping.addMethodMapping(new MethodMapping(oldMethodMapping, replacer)); } else { System.out.println(String.format("Can't map method, dropping: %s.%s %s", oldClassMapping.getDeobfName(), oldMethodMapping.getDeobfName(), oldMethodMapping.getObfSignature() )); } } return newClassMapping; } private static boolean canMigrate(Signature oldObfSignature, ClassMatches classMatches) { for (Type oldObfType : oldObfSignature.types()) { if (!canMigrate(oldObfType, classMatches)) { return false; } } return true; } private static boolean canMigrate(Type oldObfType, ClassMatches classMatches) { // non classes can be migrated if (!oldObfType.hasClass()) { return true; } // non obfuscated classes can be migrated ClassEntry classEntry = oldObfType.getClassEntry(); if (classEntry.getPackageName() != null) { return true; } // obfuscated classes with mappings can be migrated return classMatches.getUniqueMatches().containsKey(classEntry); } public static void convertMappings(Mappings mappings, BiMap changes) { // sort the changes so classes are renamed in the correct order // ie. if we have the mappings a->b, b->c, we have to apply b->c before a->b LinkedHashMap sortedChanges = Maps.newLinkedHashMap(); int numChangesLeft = changes.size(); while (!changes.isEmpty()) { Iterator> iter = changes.entrySet().iterator(); while (iter.hasNext()) { Map.Entry change = iter.next(); if (changes.containsKey(change.getValue())) { sortedChanges.put(change.getKey(), change.getValue()); iter.remove(); } } // did we remove any changes? if (numChangesLeft - changes.size() > 0) { // keep going numChangesLeft = changes.size(); } else { // can't sort anymore. There must be a loop break; } } if (!changes.isEmpty()) { throw new Error("Unable to sort class changes! There must be a cycle."); } // convert the mappings in the correct class order for (Map.Entry entry : sortedChanges.entrySet()) { mappings.renameObfClass(entry.getKey().getName(), entry.getValue().getName()); } } public interface Doer { Collection getDroppedEntries(MappingsChecker checker); Collection getObfEntries(JarIndex jarIndex); Collection> getMappings(ClassMapping destClassMapping); Set filterEntries(Collection obfEntries, T obfSourceEntry, ClassMatches classMatches); void setUpdateObfMember(ClassMapping classMapping, MemberMapping memberMapping, T newEntry); boolean hasObfMember(ClassMapping classMapping, T obfEntry); void removeMemberByObf(ClassMapping classMapping, T obfEntry); } public static Doer getFieldDoer() { return new Doer() { @Override public Collection getDroppedEntries(MappingsChecker checker) { return checker.getDroppedFieldMappings().keySet(); } @Override public Collection getObfEntries(JarIndex jarIndex) { return jarIndex.getObfFieldEntries(); } @Override public Collection> getMappings(ClassMapping destClassMapping) { return (Collection>) destClassMapping.fields(); } @Override public Set filterEntries(Collection obfDestFields, FieldEntry obfSourceField, ClassMatches classMatches) { Set out = Sets.newHashSet(); for (FieldEntry obfDestField : obfDestFields) { Type translatedDestType = translate(obfDestField.getType(), classMatches.getUniqueMatches().inverse()); if (translatedDestType.equals(obfSourceField.getType())) { out.add(obfDestField); } } return out; } @Override public void setUpdateObfMember(ClassMapping classMapping, MemberMapping memberMapping, FieldEntry newField) { FieldMapping fieldMapping = (FieldMapping) memberMapping; classMapping.setFieldObfNameAndType(fieldMapping.getObfName(), fieldMapping.getObfType(), newField.getName(), newField.getType()); } @Override public boolean hasObfMember(ClassMapping classMapping, FieldEntry obfField) { return classMapping.getFieldByObf(obfField.getName(), obfField.getType()) != null; } @Override public void removeMemberByObf(ClassMapping classMapping, FieldEntry obfField) { classMapping.removeFieldMapping(classMapping.getFieldByObf(obfField.getName(), obfField.getType())); } }; } public static Doer getMethodDoer() { return new Doer() { @Override public Collection getDroppedEntries(MappingsChecker checker) { return checker.getDroppedMethodMappings().keySet(); } @Override public Collection getObfEntries(JarIndex jarIndex) { return jarIndex.getObfBehaviorEntries(); } @Override public Collection> getMappings(ClassMapping destClassMapping) { return (Collection>) destClassMapping.methods(); } @Override public Set filterEntries(Collection obfDestFields, BehaviorEntry obfSourceField, ClassMatches classMatches) { Set out = Sets.newHashSet(); for (BehaviorEntry obfDestField : obfDestFields) { // Try to translate the signature Signature translatedDestSignature = translate(obfDestField.getSignature(), classMatches.getUniqueMatches().inverse()); if (translatedDestSignature != null && obfSourceField.getSignature() != null && translatedDestSignature.equals(obfSourceField.getSignature())) out.add(obfDestField); } return out; } @Override public void setUpdateObfMember(ClassMapping classMapping, MemberMapping memberMapping, BehaviorEntry newBehavior) { MethodMapping methodMapping = (MethodMapping) memberMapping; classMapping.setMethodObfNameAndSignature(methodMapping.getObfName(), methodMapping.getObfSignature(), newBehavior.getName(), newBehavior.getSignature()); } @Override public boolean hasObfMember(ClassMapping classMapping, BehaviorEntry obfBehavior) { return classMapping.getMethodByObf(obfBehavior.getName(), obfBehavior.getSignature()) != null; } @Override public void removeMemberByObf(ClassMapping classMapping, BehaviorEntry obfBehavior) { classMapping.removeMethodMapping(classMapping.getMethodByObf(obfBehavior.getName(), obfBehavior.getSignature())); } }; } public static int compareMethodByteCode(CodeIterator sourceIt, CodeIterator destIt) { int sourcePos = 0; int destPos = 0; while (sourceIt.hasNext() && destIt.hasNext()) { try { sourcePos = sourceIt.next(); destPos = destIt.next(); if (sourceIt.byteAt(sourcePos) != destIt.byteAt(destPos)) return sourcePos; } catch (BadBytecode badBytecode) { // Ignore bad bytecode (it might be a little bit dangerous...) } } if (sourcePos < destPos) return sourcePos; else if (destPos < sourcePos) return destPos; return sourcePos; } public static BehaviorEntry compareMethods(CtClass destCtClass, CtClass sourceCtClass, BehaviorEntry obfSourceEntry, Set obfDestEntries) { try { // Get the source method with Javassist CtMethod sourceCtClassMethod = sourceCtClass.getMethod(obfSourceEntry.getName(), obfSourceEntry.getSignature().toString()); CodeAttribute sourceAttribute = sourceCtClassMethod.getMethodInfo().getCodeAttribute(); // Empty method body, ignore! if (sourceAttribute == null) return null; for (BehaviorEntry desEntry : obfDestEntries) { try { CtMethod destCtClassMethod = destCtClass .getMethod(desEntry.getName(), desEntry.getSignature().toString()); CodeAttribute destAttribute = destCtClassMethod.getMethodInfo().getCodeAttribute(); // Ignore empty body methods if (destAttribute == null) continue; CodeIterator destIterator = destAttribute.iterator(); int maxPos = compareMethodByteCode(sourceAttribute.iterator(), destIterator); // The bytecode is identical to the original method, assuming that the method is correct! if (sourceAttribute.getCodeLength() == (maxPos + 1) && maxPos > 1) return desEntry; } catch (NotFoundException e) { e.printStackTrace(); } } } catch (NotFoundException e) { e.printStackTrace(); return null; } return null; } public static MemberMatches computeMethodsMatches(Deobfuscator destDeobfuscator, Mappings destMappings, Deobfuscator sourceDeobfuscator, Mappings sourceMappings, ClassMatches classMatches, Doer doer) { MemberMatches memberMatches = new MemberMatches<>(); // unmatched source fields are easy MappingsChecker checker = new MappingsChecker(destDeobfuscator.getJarIndex()); checker.dropBrokenMappings(destMappings); for (BehaviorEntry destObfEntry : doer.getDroppedEntries(checker)) { BehaviorEntry srcObfEntry = translate(destObfEntry, classMatches.getUniqueMatches().inverse()); memberMatches.addUnmatchedSourceEntry(srcObfEntry); } // get matched fields (anything that's left after the checks/drops is matched( for (ClassMapping classMapping : destMappings.classes()) collectMatchedFields(memberMatches, classMapping, classMatches, doer); // get unmatched dest fields doer.getObfEntries(destDeobfuscator.getJarIndex()).stream() .filter(destEntry -> !memberMatches.isMatchedDestEntry(destEntry)) .forEach(memberMatches::addUnmatchedDestEntry); // Apply mappings to deobfuscator // Create type loader TranslatingTypeLoader destTypeLoader = destDeobfuscator.createTypeLoader(); TranslatingTypeLoader sourceTypeLoader = sourceDeobfuscator.createTypeLoader(); System.out.println("Automatching " + memberMatches.getUnmatchedSourceEntries().size() + " unmatched source entries..."); // go through the unmatched source fields and try to pick out the easy matches for (ClassEntry obfSourceClass : Lists.newArrayList(memberMatches.getSourceClassesWithUnmatchedEntries())) { for (BehaviorEntry obfSourceEntry : Lists.newArrayList(memberMatches.getUnmatchedSourceEntries(obfSourceClass))) { // get the possible dest matches ClassEntry obfDestClass = classMatches.getUniqueMatches().get(obfSourceClass); // filter by type/signature Set obfDestEntries = doer.filterEntries(memberMatches.getUnmatchedDestEntries(obfDestClass), obfSourceEntry, classMatches); if (obfDestEntries.size() == 1) { // make the easy match memberMatches.makeMatch(obfSourceEntry, obfDestEntries.iterator().next()); } else if (obfDestEntries.isEmpty()) { // no match is possible =( memberMatches.makeSourceUnmatchable(obfSourceEntry, null); } else { // Multiple matches! Scan methods instructions CtClass destCtClass = destTypeLoader.loadClass(obfDestClass.getClassName()); CtClass sourceCtClass = sourceTypeLoader.loadClass(obfSourceClass.getClassName()); BehaviorEntry match = compareMethods(destCtClass, sourceCtClass, obfSourceEntry, obfDestEntries); // the method match correctly, match it on the member mapping! if (match != null) memberMatches.makeMatch(obfSourceEntry, match); } } } System.out.println(String.format("Ended up with %d ambiguous and %d unmatchable source entries", memberMatches.getUnmatchedSourceEntries().size(), memberMatches.getUnmatchableSourceEntries().size() )); return memberMatches; } public static MemberMatches computeMemberMatches(Deobfuscator destDeobfuscator, Mappings destMappings, ClassMatches classMatches, Doer doer) { MemberMatches memberMatches = new MemberMatches<>(); // unmatched source fields are easy MappingsChecker checker = new MappingsChecker(destDeobfuscator.getJarIndex()); checker.dropBrokenMappings(destMappings); for (T destObfEntry : doer.getDroppedEntries(checker)) { T srcObfEntry = translate(destObfEntry, classMatches.getUniqueMatches().inverse()); memberMatches.addUnmatchedSourceEntry(srcObfEntry); } // get matched fields (anything that's left after the checks/drops is matched( for (ClassMapping classMapping : destMappings.classes()) { collectMatchedFields(memberMatches, classMapping, classMatches, doer); } // get unmatched dest fields for (T destEntry : doer.getObfEntries(destDeobfuscator.getJarIndex())) { if (!memberMatches.isMatchedDestEntry(destEntry)) { memberMatches.addUnmatchedDestEntry(destEntry); } } System.out.println("Automatching " + memberMatches.getUnmatchedSourceEntries().size() + " unmatched source entries..."); // go through the unmatched source fields and try to pick out the easy matches for (ClassEntry obfSourceClass : Lists.newArrayList(memberMatches.getSourceClassesWithUnmatchedEntries())) { for (T obfSourceEntry : Lists.newArrayList(memberMatches.getUnmatchedSourceEntries(obfSourceClass))) { // get the possible dest matches ClassEntry obfDestClass = classMatches.getUniqueMatches().get(obfSourceClass); // filter by type/signature Set obfDestEntries = doer.filterEntries(memberMatches.getUnmatchedDestEntries(obfDestClass), obfSourceEntry, classMatches); if (obfDestEntries.size() == 1) { // make the easy match memberMatches.makeMatch(obfSourceEntry, obfDestEntries.iterator().next()); } else if (obfDestEntries.isEmpty()) { // no match is possible =( memberMatches.makeSourceUnmatchable(obfSourceEntry, null); } } } System.out.println(String.format("Ended up with %d ambiguous and %d unmatchable source entries", memberMatches.getUnmatchedSourceEntries().size(), memberMatches.getUnmatchableSourceEntries().size() )); return memberMatches; } private static void collectMatchedFields(MemberMatches memberMatches, ClassMapping destClassMapping, ClassMatches classMatches, Doer doer) { // get the fields for this class for (MemberMapping destEntryMapping : doer.getMappings(destClassMapping)) { T destObfField = destEntryMapping.getObfEntry(destClassMapping.getObfEntry()); T srcObfField = translate(destObfField, classMatches.getUniqueMatches().inverse()); memberMatches.addMatch(srcObfField, destObfField); } // recurse for (ClassMapping destInnerClassMapping : destClassMapping.innerClasses()) { collectMatchedFields(memberMatches, destInnerClassMapping, classMatches, doer); } } @SuppressWarnings("unchecked") private static T translate(T in, BiMap map) { if (in instanceof FieldEntry) { return (T) new FieldEntry( map.get(in.getClassEntry()), in.getName(), translate(((FieldEntry) in).getType(), map) ); } else if (in instanceof MethodEntry) { return (T) new MethodEntry( map.get(in.getClassEntry()), in.getName(), translate(((MethodEntry) in).getSignature(), map) ); } else if (in instanceof ConstructorEntry) { return (T) new ConstructorEntry( map.get(in.getClassEntry()), translate(((ConstructorEntry) in).getSignature(), map) ); } throw new Error("Unhandled entry type: " + in.getClass()); } private static Type translate(Type type, final BiMap map) { return new Type(type, inClassName -> { ClassEntry outClassEntry = map.get(new ClassEntry(inClassName)); if (outClassEntry == null) { return null; } return outClassEntry.getName(); }); } private static Signature translate(Signature signature, final BiMap map) { if (signature == null) { return null; } return new Signature(signature, inClassName -> { ClassEntry outClassEntry = map.get(new ClassEntry(inClassName)); if (outClassEntry == null) { return null; } return outClassEntry.getName(); }); } public static void applyMemberMatches(Mappings mappings, ClassMatches classMatches, MemberMatches memberMatches, Doer doer) { for (ClassMapping classMapping : mappings.classes()) { applyMemberMatches(classMapping, classMatches, memberMatches, doer); } } private static void applyMemberMatches(ClassMapping classMapping, ClassMatches classMatches, MemberMatches memberMatches, Doer doer) { // get the classes ClassEntry obfDestClass = new ClassEntry(classMapping.getObfFullName()); // make a map of all the renames we need to make Map renames = Maps.newHashMap(); for (MemberMapping memberMapping : Lists.newArrayList(doer.getMappings(classMapping))) { T obfOldDestEntry = memberMapping.getObfEntry(obfDestClass); T obfSourceEntry = getSourceEntryFromDestMapping(memberMapping, obfDestClass, classMatches); // but drop the unmatchable things if (memberMatches.isUnmatchableSourceEntry(obfSourceEntry)) { doer.removeMemberByObf(classMapping, obfOldDestEntry); continue; } T obfNewDestEntry = memberMatches.matches().get(obfSourceEntry); if (obfNewDestEntry != null && !obfOldDestEntry.getName().equals(obfNewDestEntry.getName())) { renames.put(obfOldDestEntry, obfNewDestEntry); } } if (!renames.isEmpty()) { // apply to this class (should never need more than n passes) int numRenamesAppliedThisRound; do { numRenamesAppliedThisRound = 0; for (MemberMapping memberMapping : Lists.newArrayList(doer.getMappings(classMapping))) { T obfOldDestEntry = memberMapping.getObfEntry(obfDestClass); T obfNewDestEntry = renames.get(obfOldDestEntry); if (obfNewDestEntry != null) { // make sure this rename won't cause a collision // otherwise, save it for the next round and try again next time if (!doer.hasObfMember(classMapping, obfNewDestEntry)) { doer.setUpdateObfMember(classMapping, memberMapping, obfNewDestEntry); renames.remove(obfOldDestEntry); numRenamesAppliedThisRound++; } } } } while (numRenamesAppliedThisRound > 0); if (!renames.isEmpty()) { System.err.println(String.format("WARNING: Couldn't apply all the renames for class %s. %d renames left.", classMapping.getObfFullName(), renames.size() )); for (Map.Entry entry : renames.entrySet()) { System.err.println(String.format("\t%s -> %s", entry.getKey().getName(), entry.getValue().getName())); } } } // recurse for (ClassMapping innerClassMapping : classMapping.innerClasses()) { applyMemberMatches(innerClassMapping, classMatches, memberMatches, doer); } } private static T getSourceEntryFromDestMapping(MemberMapping destMemberMapping, ClassEntry obfDestClass, ClassMatches classMatches) { return translate(destMemberMapping.getObfEntry(obfDestClass), classMatches.getUniqueMatches().inverse()); } }