redo name proposal (#163)

* rework interface

* work

* work

* somewhat functional

* fix and improve server stuff

* check the style

* fix rare npe due to mishandling of flatlaf

* misc

* restore old save/load behaviour

* fully working I think :D

* fix checkstyle

* port enigma-cli

* delete recaf (it broke and I'm lazy)

* fix

* error handling for invalid token types

* fix navigator issues

* clarify javadoc

* priority

* fix npe

* store proposed mappings separately

* store proposed mappings separately

* add a test for service ordering

* malformed json test

* resolve easy comments

* fix potential crash

* add more tests

* resolve todo (it didn't actually need fixing after some other changes I made)

* fix test

enigma-cli/src/main/java/org/quiltmc/enigma/command/CheckMappingsCommand.java

@@ -41,14 +41,14 @@ public static void run(Path fileJarIn, Path fileMappings) throws Exception {
 
 		for (Set<ClassEntry> partition : idx.getIndex(PackageVisibilityIndex.class).getPartitions()) {
 			long packages = partition.stream()
-					.map(project.getMapper()::deobfuscate)
+					.map(project.getRemapper()::deobfuscate)
 					.map(ClassEntry::getPackageName)
 					.distinct()
 					.count();
 			if (packages > 1) {
 				error = true;
 				Logger.error("Must be in one package:\n{}", () -> partition.stream()
-						.map(project.getMapper()::deobfuscate)
+						.map(project.getRemapper()::deobfuscate)
 						.map(ClassEntry::toString)
 						.sorted()
 						.collect(Collectors.joining("\n"))

enigma-cli/src/main/java/org/quiltmc/enigma/command/ComposeMappingsCommand.java

@@ -45,7 +45,7 @@ public String getDescription() {
 	}
 
 	public static void run(Path leftFile, Path rightFile, String resultFormat, Path resultFile, String keepMode) throws IOException, MappingParseException {
-		MappingSaveParameters saveParameters = new MappingSaveParameters(MappingFileNameFormat.BY_DEOBF);
+		MappingSaveParameters saveParameters = new MappingSaveParameters(MappingFileNameFormat.BY_DEOBF, false);
 
 		MappingFormat leftFormat = MappingFormat.parseFromFile(leftFile);
 		EntryTree<EntryMapping> left = leftFormat.read(leftFile);

enigma-cli/src/main/java/org/quiltmc/enigma/command/ConvertMappingsCommand.java

@@ -40,7 +40,7 @@ public String getDescription() {
 	}
 
 	public static void run(Path source, String resultFormat, Path output) throws MappingParseException, IOException {
-		MappingSaveParameters saveParameters = new MappingSaveParameters(MappingFileNameFormat.BY_DEOBF);
+		MappingSaveParameters saveParameters = new MappingSaveParameters(MappingFileNameFormat.BY_DEOBF, false);
 
 		MappingFormat format = MappingFormat.parseFromFile(source);
 		EntryTree<EntryMapping> mappings = format.read(source);

enigma-cli/src/main/java/org/quiltmc/enigma/command/DropInvalidMappingsCommand.java

@@ -75,6 +75,6 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO
 		}
 
 		MappingSaveParameters saveParameters = project.getEnigma().getProfile().getMappingSaveParameters();
-		format.write(project.getMapper().getObfToDeobf(), mappingsOut, ProgressListener.none(), saveParameters);
+		format.write(project.getRemapper().getMappings(), mappingsOut, ProgressListener.none(), saveParameters);
 	}
 }

enigma-cli/src/main/java/org/quiltmc/enigma/command/FillClassMappingsCommand.java

@@ -53,7 +53,7 @@ public static void run(Path jar, Path source, Path result, String resultFormat,
 		JarIndex jarIndex = loadJar(jar);
 
 		Logger.info("Reading mappings...");
-		MappingSaveParameters saveParameters = new MappingSaveParameters(MappingFileNameFormat.BY_DEOBF);
+		MappingSaveParameters saveParameters = new MappingSaveParameters(MappingFileNameFormat.BY_DEOBF, false);
 		EntryTree<EntryMapping> sourceMappings = readMappings(source, ProgressListener.none());
 
 		EntryTree<EntryMapping> resultMappings = exec(jarIndex, sourceMappings, fillAll, debug);

enigma-cli/src/main/java/org/quiltmc/enigma/command/InsertProposedMappingsCommand.java

@@ -4,28 +4,26 @@
 import org.quiltmc.enigma.api.EnigmaProfile;
 import org.quiltmc.enigma.api.EnigmaProject;
 import org.quiltmc.enigma.api.ProgressListener;
-import org.quiltmc.enigma.api.analysis.index.jar.EntryIndex;
 import org.quiltmc.enigma.api.EnigmaPlugin;
 import org.quiltmc.enigma.api.service.NameProposalService;
-import org.quiltmc.enigma.api.translation.ProposingTranslator;
-import org.quiltmc.enigma.api.translation.Translator;
 import org.quiltmc.enigma.api.translation.mapping.EntryMapping;
-import org.quiltmc.enigma.api.translation.mapping.EntryRemapper;
 import org.quiltmc.enigma.api.translation.mapping.serde.MappingSaveParameters;
 import org.quiltmc.enigma.api.translation.mapping.serde.MappingsWriter;
 import org.quiltmc.enigma.api.translation.mapping.tree.DeltaTrackingTree;
 import org.quiltmc.enigma.api.translation.mapping.tree.EntryTree;
 import org.quiltmc.enigma.api.translation.mapping.tree.HashEntryTree;
-import org.quiltmc.enigma.api.translation.representation.TypeDescriptor;
 import org.quiltmc.enigma.api.translation.representation.entry.ClassEntry;
 import org.quiltmc.enigma.api.translation.representation.entry.Entry;
 import org.quiltmc.enigma.api.translation.representation.entry.FieldEntry;
 import org.quiltmc.enigma.api.translation.representation.entry.LocalVariableEntry;
 import org.quiltmc.enigma.api.translation.representation.entry.MethodEntry;
 import org.quiltmc.enigma.util.Utils;
+import org.quiltmc.enigma.util.validation.ValidationContext;
 import org.tinylog.Logger;
 
 import java.nio.file.Path;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 import javax.annotation.Nullable;
 
 public class InsertProposedMappingsCommand extends Command {
@@ -67,85 +65,75 @@ public static void run(Path inJar, Path source, Path output, String resultFormat
 
 	public static void run(Path inJar, Path source, Path output, String resultFormat, Enigma enigma) throws Exception {
 		boolean debug = shouldDebug(new InsertProposedMappingsCommand().getName());
-		NameProposalService[] nameProposalServices = enigma.getServices().get(NameProposalService.TYPE).toArray(new NameProposalService[0]);
-		if (nameProposalServices.length == 0) {
-			Logger.error("No name proposal service found");
+		int nameProposalServices = enigma.getServices().get(NameProposalService.TYPE).size();
+		if (nameProposalServices == 0) {
+			Logger.error("No name proposal services found!");
 			return;
 		}
 
 		EnigmaProject project = openProject(inJar, source, enigma);
-		EntryTree<EntryMapping> mappings = exec(nameProposalServices, project, debug);
+		DeltaTrackingTree<EntryMapping> mappings = project.getRemapper().getMappings();
+		printStats(project);
 
 		Utils.delete(output);
-		MappingSaveParameters saveParameters = enigma.getProfile().getMappingSaveParameters();
+		MappingSaveParameters saveParameters = new MappingSaveParameters(enigma.getProfile().getMappingSaveParameters().fileNameFormat(), true);
 		MappingsWriter writer = MappingCommandsUtil.getWriter(resultFormat);
 		writer.write(mappings, output, ProgressListener.none(), saveParameters);
 
 		if (debug) {
-			writeDebugDelta((DeltaTrackingTree<EntryMapping>) mappings, output);
+			writeDebugDelta(mappings, output);
 		}
 	}
 
-	public static EntryTree<EntryMapping> exec(NameProposalService[] nameProposalServices, EnigmaProject project, boolean trackDelta) {
-		EntryTree<EntryMapping> mappings = new HashEntryTree<>(project.getMapper().getObfToDeobf());
-
-		if (trackDelta) {
-			mappings = new DeltaTrackingTree<>(mappings);
+	/**
+	 * Adds all mappings proposed by the provided services, both bytecode-based and dynamic, to the project.
+	 * @param nameProposalServices the name proposal services to run
+	 * @param project the project
+	 * @return the full mappings of the project
+	 */
+	@SuppressWarnings("unused")
+	public static EntryTree<EntryMapping> exec(NameProposalService[] nameProposalServices, EnigmaProject project) {
+		for (NameProposalService service : nameProposalServices) {
+			Map<Entry<?>, EntryMapping> jarMappings = service.getProposedNames(project.getJarIndex());
+			Map<Entry<?>, EntryMapping> dynamicMappings = service.getDynamicProposedNames(project.getRemapper(), null, null, null);
+
+			insertMappings(jarMappings, project);
+			insertMappings(dynamicMappings, project);
 		}
 
-		EntryRemapper mapper = project.getMapper();
-		Translator translator = new ProposingTranslator(mapper, nameProposalServices);
-		EntryIndex index = project.getJarIndex().getIndex(EntryIndex.class);
-
-		Logger.info("Proposing class names...");
-		int classes = 0;
-		for (ClassEntry clazz : index.getClasses()) {
-			if (insertMapping(clazz, mappings, mapper, translator)) {
-				classes++;
-			}
-		}
+		return project.getRemapper().getMappings();
+	}
 
-		Logger.info("Proposing field names...");
-		int fields = 0;
-		for (FieldEntry field : index.getFields()) {
-			if (insertMapping(field, mappings, mapper, translator)) {
-				fields++;
+	private static void insertMappings(@Nullable Map<Entry<?>, EntryMapping> mappings, EnigmaProject project) {
+		if (mappings != null) {
+			for (var entry : mappings.entrySet()) {
+				if (entry.getValue() != null) {
+					project.getRemapper().putMapping(new ValidationContext(null), entry.getKey(), entry.getValue());
+				}
 			}
 		}
+	}
 
-		Logger.info("Proposing method and parameter names...");
-		int methods = 0;
-		int parameters = 0;
-		for (MethodEntry method : index.getMethods()) {
-			if (insertMapping(method, mappings, mapper, translator)) {
-				methods++;
-			}
-
-			int p = index.getMethodAccess(method).isStatic() ? 0 : 1;
-			for (TypeDescriptor paramDesc : method.getDesc().getArgumentDescs()) {
-				LocalVariableEntry param = new LocalVariableEntry(method, p, "", true, null);
-				if (insertMapping(param, mappings, mapper, translator)) {
-					parameters++;
+	public static void printStats(EnigmaProject project) {
+		EntryTree<EntryMapping> mappings = new HashEntryTree<>(project.getRemapper().getProposedMappings());
+		AtomicInteger classes = new AtomicInteger();
+		AtomicInteger fields = new AtomicInteger();
+		AtomicInteger methods = new AtomicInteger();
+		AtomicInteger parameters = new AtomicInteger();
+		mappings.forEach(node -> {
+			if (node.getValue() != null) {
+				if (node.getEntry() instanceof ClassEntry) {
+					classes.incrementAndGet();
+				} else if (node.getEntry() instanceof FieldEntry) {
+					fields.incrementAndGet();
+				} else if (node.getEntry() instanceof MethodEntry) {
+					methods.incrementAndGet();
+				} else if (node.getEntry() instanceof LocalVariableEntry) {
+					parameters.incrementAndGet();
 				}
-
-				p += paramDesc.getSize();
 			}
-		}
+		});
 
 		Logger.info("Proposed names for {} classes, {} fields, {} methods, {} parameters!", classes, fields, methods, parameters);
-		return mappings;
-	}
-
-	private static <T extends Entry<?>> boolean insertMapping(T entry, EntryTree<EntryMapping> mappings, EntryRemapper mapper, Translator translator) {
-		T deobf = mapper.extendedDeobfuscate(entry).getValue();
-		String name = translator.extendedTranslate(entry).getValue().getName();
-		if (!deobf.getName().equals(name) && !entry.getName().equals(name)) {
-			String javadoc = deobf.getJavadocs();
-			EntryMapping mapping = javadoc != null && !javadoc.isEmpty() ? new EntryMapping(name, javadoc) : new EntryMapping(name);
-			mappings.insert(entry, mapping);
-			return true;
-		}
-
-		return false;
 	}
 }

enigma-cli/src/main/java/org/quiltmc/enigma/command/InvertMappingsCommand.java

@@ -40,7 +40,7 @@ public String getDescription() {
 	}
 
 	public static void run(Path sourceFile, String resultFormat, Path resultFile) throws MappingParseException, IOException {
-		MappingSaveParameters saveParameters = new MappingSaveParameters(MappingFileNameFormat.BY_DEOBF);
+		MappingSaveParameters saveParameters = new MappingSaveParameters(MappingFileNameFormat.BY_DEOBF, false);
 		MappingFormat format = MappingFormat.parseFromFile(sourceFile);
 
 		EntryTree<EntryMapping> source = format.read(sourceFile);

enigma-cli/src/main/java/org/quiltmc/enigma/command/MapSpecializedMethodsCommand.java

@@ -53,7 +53,7 @@ public static void run(Path jar, Path sourcePath, String resultFormat, Path outp
 		boolean debug = shouldDebug(new MapSpecializedMethodsCommand().getName());
 		JarIndex jarIndex = loadJar(jar);
 
-		MappingSaveParameters saveParameters = new MappingSaveParameters(MappingFileNameFormat.BY_DEOBF);
+		MappingSaveParameters saveParameters = new MappingSaveParameters(MappingFileNameFormat.BY_DEOBF, false);
 		MappingFormat sourceFormat = MappingFormat.parseFromFile(sourcePath);
 		EntryTree<EntryMapping> source = sourceFormat.read(sourcePath);
 

enigma-server/docs/protocol.md

@@ -213,12 +213,20 @@ struct entry_change {
     if <javadoc_change == TRISTATE_CHANGE_SET> {
         utf javadoc;
     }
+    if <token_type_change == TRISTATE_CHANGE_SET> {
+        unsigned short token_type;
+    }
+    if <source_plugin_id_change == TRISTATE_CHANGE_SET> {
+        utf source_plugin_id;
+    }
 }
 ```
 - `entry`: The entry this change gets applied to.
 - `flags`: See definition of `entry_change_flags`.
 - `deobf_name`: The new deobfuscated name, if deobf_name_change == TRISTATE_CHANGE_SET
 - `javadoc`: The new javadoc, if javadoc_change == TRISTATE_CHANGE_SET
+- `token_type`: The new token type, if token_type_change == TRISTATE_CHANGE_SET
+- `source_plugin_id`: The ID of the plugin that proposed this entry's name, if source_plugin_id_change == TRISTATE_CHANGE_SET
 
 ### Login (client-to-server)
 ```c
@@ -278,9 +286,10 @@ struct SyncMappingsS2CPacket {
 }
 struct MappingNode {
     NoParentEntry obf_entry;
-    boolean is_named;
     utf name;
     utf javadoc;
+    unsigned short token_type;
+    utf source_plugin_id;
     unsigned short children_count;
     MappingNode children[children_count];
 }
@@ -290,7 +299,9 @@ typedef { Entry but without the has_parent or parent fields } NoParentEntry;
 - `obf_entry`: The value of a node, containing the obfuscated name and descriptor of the entry.
 - `name`: The deobfuscated name of the entry, if it exists, otherwise the empty string.
 - `javadoc`: The documentation for the entry, if it exists, otherwise the empty string.
-- `children`: The children of this node
+- `token_type`: The token type of the entry.
+- `source_plugin_id`: If the name is proposed, the ID of the plugin that proposed the name, otherwise the empty string.
+- `children`: The children of this node.
 
 ### Message (server-to-client)
 ```c

enigma-server/src/main/java/org/quiltmc/enigma/network/DedicatedEnigmaServer.java

@@ -8,6 +8,7 @@
 import org.quiltmc.enigma.api.translation.mapping.serde.MappingParseException;
 import org.quiltmc.enigma.api.translation.mapping.EntryRemapper;
 import org.quiltmc.enigma.api.translation.mapping.serde.MappingFormat;
+import org.quiltmc.enigma.api.translation.mapping.tree.HashEntryTree;
 import org.quiltmc.enigma.util.Utils;
 import joptsimple.OptionParser;
 import joptsimple.OptionSet;
@@ -117,10 +118,10 @@ public static void main(String[] args) {
 			MappingFormat mappingFormat = MappingFormat.parseFromFile(mappingsFile);
 			EntryRemapper mappings;
 			if (!Files.exists(mappingsFile)) {
-				mappings = EntryRemapper.empty(project.getJarIndex());
+				mappings = EntryRemapper.mapped(project.getJarIndex(), project.getMappingsIndex(), project.getRemapper().getProposedMappings(), new HashEntryTree<>(), enigma.getNameProposalServices());
 			} else {
 				Logger.info("Reading mappings...");
-				mappings = EntryRemapper.mapped(project.getJarIndex(), project.getMappingsIndex(), mappingFormat.read(mappingsFile));
+				mappings = EntryRemapper.mapped(project.getJarIndex(), project.getMappingsIndex(), project.getRemapper().getProposedMappings(), mappingFormat.read(mappingsFile), enigma.getNameProposalServices());
 			}
 
 			PrintWriter log = new PrintWriter(Files.newBufferedWriter(logFile));
@@ -134,9 +135,7 @@ public static void main(String[] args) {
 			return;
 		}
 
-		// noinspection RedundantSuppression
-		// noinspection Convert2MethodRef - javac 8 bug
-		Executors.newScheduledThreadPool(1).scheduleAtFixedRate(() -> server.runOnThread(() -> server.saveMappings()), 0, 1, TimeUnit.MINUTES);
+		Executors.newScheduledThreadPool(1).scheduleAtFixedRate(() -> server.runOnThread(server::saveMappings), 0, 1, TimeUnit.MINUTES);
 		Runtime.getRuntime().addShutdownHook(new Thread(server::saveMappings));
 
 		while (true) {
@@ -155,7 +154,7 @@ public synchronized void stop() {
 	}
 
 	private void saveMappings() {
-		this.mappingFormat.write(this.getMappings().getObfToDeobf(), this.getMappings().takeMappingDelta(), this.mappingsFile, ProgressListener.none(), this.profile.getMappingSaveParameters());
+		this.mappingFormat.write(this.getRemapper().getMappings(), this.getRemapper().takeMappingDelta(), this.mappingsFile, ProgressListener.none(), this.profile.getMappingSaveParameters());
 		this.log.flush();
 	}
 

enigma-server/src/main/java/org/quiltmc/enigma/network/EnigmaServer.java

@@ -47,18 +47,18 @@ public abstract class EnigmaServer {
 	private final char[] password;
 
 	public static final int DUMMY_SYNC_ID = 0;
-	private final EntryRemapper mappings;
+	private final EntryRemapper remapper;
 	private final Map<Entry<?>, Integer> syncIds = new HashMap<>();
 	private final Map<Integer, Entry<?>> inverseSyncIds = new HashMap<>();
 	private final Map<Integer, Set<Socket>> clientsNeedingConfirmation = new HashMap<>();
 	private int nextSyncId = DUMMY_SYNC_ID + 1;
 
 	private static int nextIoId = 0;
 
-	public EnigmaServer(byte[] jarChecksum, char[] password, EntryRemapper mappings, int port) {
+	public EnigmaServer(byte[] jarChecksum, char[] password, EntryRemapper remapper, int port) {
 		this.jarChecksum = jarChecksum;
 		this.password = password;
-		this.mappings = mappings;
+		this.remapper = remapper;
 		this.port = port;
 	}
 
@@ -258,8 +258,8 @@ public void confirmChange(Socket client, int syncId) {
 		}
 	}
 
-	public void sendCorrectMapping(Socket client, Entry<?> entry, boolean refreshClassTree) {
-		EntryMapping oldMapping = this.mappings.getDeobfMapping(entry);
+	public void sendCorrectMapping(Socket client, Entry<?> entry) {
+		EntryMapping oldMapping = this.remapper.getMapping(entry);
 		String oldName = oldMapping.targetName();
 		if (oldName == null) {
 			this.sendPacket(client, new EntryChangeS2CPacket(DUMMY_SYNC_ID, EntryChange.modify(entry).clearDeobfName()));
@@ -286,8 +286,8 @@ public char[] getPassword() {
 		return this.password;
 	}
 
-	public EntryRemapper getMappings() {
-		return this.mappings;
+	public EntryRemapper getRemapper() {
+		return this.remapper;
 	}
 
 	public void sendMessage(ServerMessage message) {

enigma-server/src/main/java/org/quiltmc/enigma/network/packet/EntryChangeC2SPacket.java

@@ -37,12 +37,12 @@ public void handle(ServerPacketHandler handler) {
 		boolean valid = handler.server().canModifyEntry(handler.client(), this.change.getTarget());
 
 		if (valid) {
-			EntryUtil.applyChange(vc, handler.server().getMappings(), this.change);
+			EntryUtil.applyChange(vc, handler.server().getRemapper(), this.change);
 			valid = vc.canProceed();
 		}
 
 		if (!valid) {
-			handler.server().sendCorrectMapping(handler.client(), this.change.getTarget(), true);
+			handler.server().sendCorrectMapping(handler.client(), this.change.getTarget());
 			return;
 		}
 

enigma-server/src/main/java/org/quiltmc/enigma/network/packet/LoginC2SPacket.java

@@ -72,7 +72,7 @@ public void handle(ServerPacketHandler handler) {
 			return;
 		}
 
-		handler.server().sendPacket(handler.client(), new SyncMappingsS2CPacket(handler.server().getMappings().getObfToDeobf()));
+		handler.server().sendPacket(handler.client(), new SyncMappingsS2CPacket(handler.server().getRemapper().getMappings()));
 		handler.server().sendMessage(ServerMessage.connect(this.username));
 	}
 }