From d3fc0b55515e81ae1b10fa16129f05b0241271f0 Mon Sep 17 00:00:00 2001 From: jeff Date: Sun, 31 Aug 2014 14:41:24 -0400 Subject: fixed lots of bugs in the mappings converter. It's finally ready. =) --- src/cuchaz/enigma/convert/ClassIdentity.java | 23 +-- src/cuchaz/enigma/convert/ClassMatcher.java | 260 ++++++++++++++++----------- src/cuchaz/enigma/convert/ClassMatching.java | 26 +-- src/cuchaz/enigma/mapping/Mappings.java | 29 ++- 4 files changed, 194 insertions(+), 144 deletions(-) (limited to 'src') diff --git a/src/cuchaz/enigma/convert/ClassIdentity.java b/src/cuchaz/enigma/convert/ClassIdentity.java index 8de7128..bd2824b 100644 --- a/src/cuchaz/enigma/convert/ClassIdentity.java +++ b/src/cuchaz/enigma/convert/ClassIdentity.java @@ -60,7 +60,6 @@ import cuchaz.enigma.mapping.SignatureUpdater.ClassNameUpdater; public class ClassIdentity { private ClassEntry m_classEntry; - private String m_rawName; private SidedClassNamer m_namer; private Multiset m_fields; private Multiset m_methods; @@ -71,18 +70,13 @@ public class ClassIdentity private Multiset m_implementations; private Multiset m_references; - public ClassIdentity( CtClass c, SidedClassNamer namer, JarIndex index, boolean useReferences, boolean useRawNames ) + public ClassIdentity( CtClass c, SidedClassNamer namer, JarIndex index, boolean useReferences ) { m_namer = namer; // stuff from the bytecode m_classEntry = new ClassEntry( Descriptor.toJvmName( c.getName() ) ); - m_rawName = ""; - if( useRawNames ) - { - m_rawName = m_classEntry.getName(); - } m_fields = HashMultiset.create(); for( CtField field : c.getDeclaredFields() ) { @@ -186,12 +180,6 @@ public class ClassIdentity buf.append( " " ); buf.append( hashCode() ); buf.append( "\n" ); - if( m_rawName.length() > 0 ) - { - buf.append( "\traw name: " ); - buf.append( m_rawName ); - buf.append( "\n" ); - } for( String field : m_fields ) { buf.append( "\tfield " ); @@ -439,8 +427,7 @@ public class ClassIdentity public boolean equals( ClassIdentity other ) { - return m_rawName.equals( other.m_rawName ) - && m_fields.equals( other.m_fields ) + return m_fields.equals( other.m_fields ) && m_methods.equals( other.m_methods ) && m_constructors.equals( other.m_constructors ) && m_staticInitializer.equals( other.m_staticInitializer ) @@ -454,7 +441,6 @@ public class ClassIdentity public int hashCode( ) { List objs = Lists.newArrayList(); - objs.add( m_rawName ); objs.addAll( m_fields ); objs.addAll( m_methods ); objs.addAll( m_constructors ); @@ -473,6 +459,11 @@ public class ClassIdentity + getNumMatches( m_constructors, other.m_constructors ); } + public int getMaxMatchScore( ) + { + return m_fields.size() + m_methods.size() + m_constructors.size(); + } + public boolean matches( CtClass c ) { // just compare declaration counts diff --git a/src/cuchaz/enigma/convert/ClassMatcher.java b/src/cuchaz/enigma/convert/ClassMatcher.java index ac07a5b..b551da2 100644 --- a/src/cuchaz/enigma/convert/ClassMatcher.java +++ b/src/cuchaz/enigma/convert/ClassMatcher.java @@ -14,17 +14,14 @@ import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; -import java.util.AbstractMap; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.TreeMap; import java.util.jar.JarFile; import javassist.CtBehavior; @@ -32,9 +29,11 @@ import javassist.CtClass; import com.beust.jcommander.internal.Lists; import com.beust.jcommander.internal.Sets; +import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import com.google.common.collect.Maps; +import com.google.common.collect.Multimap; import cuchaz.enigma.TranslatingTypeLoader; import cuchaz.enigma.analysis.JarIndex; @@ -55,22 +54,28 @@ public class ClassMatcher { // TEMP JarFile sourceJar = new JarFile( new File( "input/1.8-pre1.jar" ) ); - JarFile destJar = new JarFile( new File( "input/1.8-pre2.jar" ) ); + JarFile destJar = new JarFile( new File( "input/1.8-pre3.jar" ) ); File inMappingsFile = new File( "../minecraft-mappings/1.8-pre.mappings" ); - File outMappingsFile = new File( "../minecraft-mappings/1.8-pre2.mappings" ); + File outMappingsFile = new File( "../minecraft-mappings/1.8-pre3.mappings" ); + + // define a matching to use when the automated system cannot find a match + Map fallbackMatching = Maps.newHashMap(); + fallbackMatching.put( "none/ayb", "none/ayb" ); + fallbackMatching.put( "none/ayd", "none/ayd" ); + fallbackMatching.put( "none/bgk", "none/bgk" ); // do the conversion Mappings mappings = new MappingsReader().read( new FileReader( inMappingsFile ) ); - convertMappings( sourceJar, destJar, mappings ); + convertMappings( sourceJar, destJar, mappings, fallbackMatching ); - // write out the convert mappings + // write out the converted mappings FileWriter writer = new FileWriter( outMappingsFile ); new MappingsWriter().write( writer, mappings ); writer.close(); System.out.println( "Wrote converted mappings to:\n\t" + outMappingsFile.getAbsolutePath() ); } - private static void convertMappings( JarFile sourceJar, JarFile destJar, Mappings mappings ) + private static void convertMappings( JarFile sourceJar, JarFile destJar, Mappings mappings, Map fallbackMatching ) { // index jars System.out.println( "Indexing source jar..." ); @@ -83,58 +88,77 @@ public class ClassMatcher TranslatingTypeLoader destLoader = new TranslatingTypeLoader( destJar, destIndex ); // compute the matching - ClassMatching matching = ClassMatcher.computeMatching( sourceIndex, sourceLoader, destIndex, destLoader ); + ClassMatching matching = computeMatching( sourceIndex, sourceLoader, destIndex, destLoader ); // start the class conversion map with the unique and ambiguous matchings Map>> conversionMap = matching.getConversionMap(); - // probabilistically match the unmatched source classes - for( ClassIdentity sourceClass : new ArrayList( matching.getUnmatchedSourceClasses() ) ) + // get all the obf class names used in the mappings + Set usedClassNames = mappings.getAllObfClassNames(); + Set allClassNames = Sets.newHashSet(); + for( ClassEntry classEntry : sourceIndex.getObfClassEntries() ) + { + allClassNames.add( classEntry.getName() ); + } + usedClassNames.retainAll( allClassNames ); + + // probabilistically match the non-uniquely-matched source classes + for( Map.Entry> entry : conversionMap.values() ) { + ClassIdentity sourceClass = entry.getKey(); + List destClasses = entry.getValue(); + + // skip classes that are uniquely matched + if( destClasses.size() == 1 ) + { + continue; + } + + // skip classes that aren't used in the mappings + if( !usedClassNames.contains( sourceClass.getClassEntry().getName() ) ) + { + continue; + } + System.out.println( "No exact match for source class " + sourceClass.getClassEntry() ); // find the closest classes - TreeMap scoredMatches = Maps.newTreeMap( Collections.reverseOrder() ); - for( ClassIdentity c : matching.getUnmatchedDestClasses() ) + Multimap scoredMatches = ArrayListMultimap.create(); + for( ClassIdentity c : destClasses ) { scoredMatches.put( sourceClass.getMatchScore( c ), c ); } - Iterator> iter = scoredMatches.entrySet().iterator(); - for( int i=0; i<10 && iter.hasNext(); i++ ) - { - Map.Entry score = iter.next(); - System.out.println( String.format( "\tScore: %3d %s", score.getKey(), score.getValue().getClassEntry().getName() ) ); - } + List scores = new ArrayList( scoredMatches.keySet() ); + Collections.sort( scores, Collections.reverseOrder() ); + printScoredMatches( sourceClass.getMaxMatchScore(), scores, scoredMatches ); // does the best match have a non-zero score and the same name? - Map.Entry bestMatch = scoredMatches.firstEntry(); - if( bestMatch.getKey() > 0 && bestMatch.getValue().getClassEntry().equals( sourceClass.getClassEntry() ) ) + int bestScore = scores.get( 0 ); + Collection bestMatches = scoredMatches.get( bestScore ); + if( bestScore > 0 && bestMatches.size() == 1 ) { - // use it - System.out.println( "\tAutomatically choosing likely match: " + bestMatch.getValue().getClassEntry().getName() ); - conversionMap.put( - sourceClass.getClassEntry().getName(), - new AbstractMap.SimpleEntry>( sourceClass, Arrays.asList( bestMatch.getValue() ) ) - ); + ClassIdentity bestMatch = bestMatches.iterator().next(); + if( bestMatch.getClassEntry().equals( sourceClass.getClassEntry() ) ) + { + // use it + System.out.println( "\tAutomatically choosing likely match: " + bestMatch.getClassEntry().getName() ); + destClasses.clear(); + destClasses.add( bestMatch ); + } } } // use the matching to convert the mappings BiMap classConversion = HashBiMap.create(); Set unmatchedSourceClasses = Sets.newHashSet(); - for( String className : mappings.getAllObfClassNames() ) + for( String className : usedClassNames ) { // is there a match for this class? Map.Entry> entry = conversionMap.get( className ); ClassIdentity sourceClass = entry.getKey(); List matches = entry.getValue(); - if( matches.isEmpty() ) - { - // no match! =( - unmatchedSourceClasses.add( className ); - } - else if( matches.size() == 1 ) + if( matches.size() == 1 ) { // unique match! We're good to go! classConversion.put( @@ -142,10 +166,21 @@ public class ClassMatcher matches.get( 0 ).getClassEntry().getName() ); } - else if( matches.size() > 1 ) + else { - // too many matches! =( - unmatchedSourceClasses.add( className ); + // no match, check the fallback matching + String fallbackMatch = fallbackMatching.get( className ); + if( fallbackMatch != null ) + { + classConversion.put( + sourceClass.getClassEntry().getName(), + fallbackMatch + ); + } + else + { + unmatchedSourceClasses.add( className ); + } } } @@ -176,17 +211,11 @@ public class ClassMatcher } } - // TEMP: show some classes - for( String className : Arrays.asList( "none/em", "none/ej", "none/en" ) ) - { - System.out.println( String.format( "check: %s -> %s", className, classConversion.get( className ) ) ); - } - - // convert the classes + // convert the mappings mappings.renameObfClasses( classConversion ); - // look for method matches - System.out.println( "Matching methods..." ); + // check the method matches + System.out.println( "Checking methods..." ); for( ClassMapping classMapping : mappings.classes() ) { ClassEntry classEntry = new ClassEntry( classMapping.getObfName() ); @@ -234,91 +263,93 @@ public class ClassMatcher } } } + + System.out.println( "Done!" ); } public static ClassMatching computeMatching( JarIndex sourceIndex, TranslatingTypeLoader sourceLoader, JarIndex destIndex, TranslatingTypeLoader destLoader ) { System.out.println( "Matching classes..." ); ClassMatching matching = null; - for( boolean useRawNames : Arrays.asList( false/*, true*/ ) ) + for( boolean useReferences : Arrays.asList( false, true ) ) { - for( boolean useReferences : Arrays.asList( false, true ) ) + int numMatches = 0; + do { - int numMatches = 0; - do + SidedClassNamer sourceNamer = null; + SidedClassNamer destNamer = null; + if( matching != null ) { - SidedClassNamer sourceNamer = null; - SidedClassNamer destNamer = null; - if( matching != null ) - { - // build a class namer - ClassNamer namer = new ClassNamer( matching.getUniqueMatches() ); - sourceNamer = namer.getSourceNamer(); - destNamer = namer.getDestNamer(); - - // note the number of matches - numMatches = matching.getUniqueMatches().size(); - } + // build a class namer + ClassNamer namer = new ClassNamer( matching.getUniqueMatches() ); + sourceNamer = namer.getSourceNamer(); + destNamer = namer.getDestNamer(); - // get the entries left to match - Set sourceClassEntries = sourceIndex.getObfClassEntries(); - Set destClassEntries = destIndex.getObfClassEntries(); - if( matching != null ) + // note the number of matches + numMatches = matching.getUniqueMatches().size(); + } + + // get the entries left to match + Set sourceClassEntries = Sets.newHashSet(); + Set destClassEntries = Sets.newHashSet(); + if( matching == null ) + { + sourceClassEntries.addAll( sourceIndex.getObfClassEntries() ); + destClassEntries.addAll( destIndex.getObfClassEntries() ); + matching = new ClassMatching(); + } + else + { + for( Map.Entry,List> entry : matching.getAmbiguousMatches().entrySet() ) { - sourceClassEntries.clear(); - destClassEntries.clear(); - for( Map.Entry,List> entry : matching.getAmbiguousMatches().entrySet() ) - { - for( ClassIdentity c : entry.getKey() ) - { - sourceClassEntries.add( c.getClassEntry() ); - matching.removeSource( c ); - } - for( ClassIdentity c : entry.getValue() ) - { - destClassEntries.add( c.getClassEntry() ); - matching.removeDest( c ); - } - } - for( ClassIdentity c : matching.getUnmatchedSourceClasses() ) + for( ClassIdentity c : entry.getKey() ) { sourceClassEntries.add( c.getClassEntry() ); matching.removeSource( c ); } - for( ClassIdentity c : matching.getUnmatchedDestClasses() ) + for( ClassIdentity c : entry.getValue() ) { destClassEntries.add( c.getClassEntry() ); matching.removeDest( c ); } } - else + for( ClassIdentity c : matching.getUnmatchedSourceClasses() ) { - matching = new ClassMatching(); + sourceClassEntries.add( c.getClassEntry() ); + matching.removeSource( c ); } - - // compute a matching for the classes - for( ClassEntry classEntry : sourceClassEntries ) - { - CtClass c = sourceLoader.loadClass( classEntry.getName() ); - ClassIdentity sourceClass = new ClassIdentity( c, sourceNamer, sourceIndex, useReferences, useRawNames ); - matching.addSource( sourceClass ); - } - for( ClassEntry classEntry : destClassEntries ) + for( ClassIdentity c : matching.getUnmatchedDestClasses() ) { - CtClass c = destLoader.loadClass( classEntry.getName() ); - ClassIdentity destClass = new ClassIdentity( c, destNamer, destIndex, useReferences, useRawNames ); - matching.matchDestClass( destClass ); + destClassEntries.add( c.getClassEntry() ); + matching.removeDest( c ); } - - // TEMP - System.out.println( matching ); } - while( matching.getUniqueMatches().size() - numMatches > 0 ); + + // compute a matching for the classes + for( ClassEntry classEntry : sourceClassEntries ) + { + CtClass c = sourceLoader.loadClass( classEntry.getName() ); + ClassIdentity sourceClass = new ClassIdentity( c, sourceNamer, sourceIndex, useReferences ); + matching.addSource( sourceClass ); + } + for( ClassEntry classEntry : destClassEntries ) + { + CtClass c = destLoader.loadClass( classEntry.getName() ); + ClassIdentity destClass = new ClassIdentity( c, destNamer, destIndex, useReferences ); + matching.matchDestClass( destClass ); + } + + // TEMP + System.out.println( matching ); } + while( matching.getUniqueMatches().size() - numMatches > 0 ); } - // DEBUG: check the class matches + // check the class matches System.out.println( "Checking class matches..." ); + ClassNamer namer = new ClassNamer( matching.getUniqueMatches() ); + SidedClassNamer sourceNamer = namer.getSourceNamer(); + SidedClassNamer destNamer = namer.getDestNamer(); for( Map.Entry entry : matching.getUniqueMatches().entrySet() ) { // check source @@ -327,7 +358,7 @@ public class ClassMatcher assert( sourceC != null ) : "Unable to load source class " + sourceClass.getClassEntry(); assert( sourceClass.matches( sourceC ) ) - : "Source " + sourceClass + " doesn't match " + new ClassIdentity( sourceC, null, sourceIndex, false, false ); + : "Source " + sourceClass + " doesn't match " + new ClassIdentity( sourceC, sourceNamer, sourceIndex, false ); // check dest ClassIdentity destClass = entry.getValue(); @@ -335,7 +366,7 @@ public class ClassMatcher assert( destC != null ) : "Unable to load dest class " + destClass.getClassEntry(); assert( destClass.matches( destC ) ) - : "Dest " + destClass + " doesn't match " + new ClassIdentity( destC, null, destIndex, false, false ); + : "Dest " + destClass + " doesn't match " + new ClassIdentity( destC, destNamer, destIndex, false ); } // warn about the ambiguous matchings @@ -372,6 +403,27 @@ public class ClassMatcher return matching; } + private static void printScoredMatches( int maxScore, List scores, Multimap scoredMatches ) + { + int numScoredMatchesShown = 0; + for( int score : scores ) + { + for( ClassIdentity scoredMatch : scoredMatches.get( score ) ) + { + System.out.println( String.format( "\tScore: %3d %3.0f%% %s", + score, + 100.0*score/maxScore, + scoredMatch.getClassEntry().getName() + ) ); + + if( numScoredMatchesShown++ > 10 ) + { + return; + } + } + } + } + private static List getClassNames( Collection classes ) { List out = Lists.newArrayList(); diff --git a/src/cuchaz/enigma/convert/ClassMatching.java b/src/cuchaz/enigma/convert/ClassMatching.java index ef5a7d8..6ce8c88 100644 --- a/src/cuchaz/enigma/convert/ClassMatching.java +++ b/src/cuchaz/enigma/convert/ClassMatching.java @@ -183,7 +183,7 @@ public class ClassMatching { conversion.put( sourceClass.getClassEntry().getName(), - new AbstractMap.SimpleEntry>( sourceClass, new ArrayList() ) + new AbstractMap.SimpleEntry>( sourceClass, getUnmatchedDestClasses() ) ); } return conversion; @@ -193,25 +193,11 @@ public class ClassMatching public String toString( ) { StringBuilder buf = new StringBuilder(); - - buf.append( "Source classes: " ); - buf.append( getSourceClasses().size() ); - buf.append( "\n\tUnique: " ); - buf.append( getUniqueMatches().size() ); - buf.append( "\n\tAmbiguous: " ); - buf.append( getNumAmbiguousSourceMatches() ); - buf.append( "\n\tUnmatched: " ); - buf.append( getUnmatchedSourceClasses().size() ); - - buf.append( "\nDest classes: " ); - buf.append( getDestClasses().size() ); - buf.append( "\n\tUnique: " ); - buf.append( getUniqueMatches().size() ); - buf.append( "\n\tAmbiguous: " ); - buf.append( getNumAmbiguousDestMatches() ); - buf.append( "\n\tUnmatched: " ); - buf.append( getUnmatchedDestClasses().size() ); - + buf.append( String.format( "%12s%8s%8s\n", "", "Source", "Dest" ) ); + buf.append( String.format( "%12s%8d%8d\n", "Classes", getSourceClasses().size(), getDestClasses().size() ) ); + buf.append( String.format( "%12s%8d%8d\n", "Unique", getUniqueMatches().size(), getUniqueMatches().size() ) ); + buf.append( String.format( "%12s%8d%8d\n", "Ambiguous", getNumAmbiguousSourceMatches(), getNumAmbiguousDestMatches() ) ); + buf.append( String.format( "%12s%8d%8d\n", "Unmatched", getUnmatchedSourceClasses().size(), getUnmatchedDestClasses().size() ) ); return buf.toString(); } } diff --git a/src/cuchaz/enigma/mapping/Mappings.java b/src/cuchaz/enigma/mapping/Mappings.java index c92f8de..4b47d16 100644 --- a/src/cuchaz/enigma/mapping/Mappings.java +++ b/src/cuchaz/enigma/mapping/Mappings.java @@ -15,15 +15,17 @@ import java.io.InputStream; import java.io.ObjectInputStream; import java.io.Serializable; import java.util.ArrayList; -import java.util.List; import java.util.Map; +import java.util.Set; import java.util.zip.GZIPInputStream; +import com.beust.jcommander.internal.Sets; import com.google.common.collect.Maps; import cuchaz.enigma.Util; import cuchaz.enigma.analysis.Ancestries; import cuchaz.enigma.analysis.DeobfuscatedAncestries; +import cuchaz.enigma.mapping.SignatureUpdater.ClassNameUpdater; public class Mappings implements Serializable { @@ -163,9 +165,28 @@ public class Mappings implements Serializable } } - public List getAllObfClassNames( ) + public Set getAllObfClassNames( ) { - // TODO: implement this - return null; + final Set classNames = Sets.newHashSet(); + for( ClassMapping classMapping : classes() ) + { + // add the class name + classNames.add( classMapping.getObfName() ); + + // add classes from method signatures + for( MethodMapping methodMapping : classMapping.methods() ) + { + SignatureUpdater.update( methodMapping.getObfSignature(), new ClassNameUpdater( ) + { + @Override + public String update( String className ) + { + classNames.add( className ); + return className; + } + } ); + } + } + return classNames; } } -- cgit v1.2.3