Much faster recipe dumper (#920)

Co-authored-by: slprime <31038811+slprime@users.noreply.github.com>

src/main/java/codechicken/nei/commands/CommandRecipeId.java

@@ -8,13 +8,15 @@ import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.TreeSet;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
 import net.minecraft.command.CommandBase;
@@ -39,6 +41,7 @@ import codechicken.nei.recipe.GuiRecipeTab;
 import codechicken.nei.recipe.ICraftingHandler;
 import codechicken.nei.recipe.Recipe;
 import codechicken.nei.recipe.RecipeHandlerQuery;
+import codechicken.nei.recipe.TemplateRecipeHandler;
 import codechicken.nei.util.NBTJson;
 
 public class CommandRecipeId extends CommandBase {
@@ -125,12 +128,16 @@ public class CommandRecipeId extends CommandBase {
 
         private ArrayList<ICraftingHandler> craftinghandlers = new ArrayList<>();
         private ArrayList<ICraftingHandler> serialCraftingHandlers = new ArrayList<>();
+
         protected final ICommandSender sender;
         protected final File currFile;
+        protected final boolean fastDump;
 
-        public ProcessDumpThread(ICommandSender sender, File currFile) {
+        public ProcessDumpThread(ICommandSender sender, File currFile, boolean fastDump) {
             this.sender = sender;
             this.currFile = currFile;
+            this.fastDump = fastDump;
+
             ClientHandler.loadSettingsFile("recipeidblacklist.cfg", lines -> {
                 final Set<String> names = lines.collect(Collectors.toSet());
 
@@ -150,38 +157,10 @@ public class CommandRecipeId extends CommandBase {
             sendChatInfoMessage(sender, "nei.chat.recipeid.dump.start");
 
             try (BufferedWriter writer = Files.newBufferedWriter(currFile.toPath(), StandardCharsets.UTF_8)) {
-                int total = ItemList.items.size();
-                int count = 0;
-
-                for (ItemStack stack : ItemList.items) {
-
-                    if (count % 1000 == 0) {
-                        NEIClientConfig.logger.info(
-                                "({}/{}). Processing {} crafting recipes...",
-                                count,
-                                total,
-                                stack.getDisplayName());
-                    }
-
-                    count++;
-
-                    for (ICraftingHandler handler : getCraftingHandlers(stack)) {
-                        for (int index = 0, num = handler.numRecipes(); index < num; index++) {
-                            try {
-                                final Recipe recipe = Recipe.of(handler, index);
-                                if (!recipe.getIngredients().isEmpty() && !recipe.getResults().isEmpty()) {
-                                    writer.write(NBTJson.toJson(recipe.getRecipeId().toJsonObject()));
-                                    writer.newLine();
-                                }
-                            } catch (Exception ex) {
-                                NEIClientConfig.logger.error(
-                                        "Found Broken RecipeId {}:{}",
-                                        GuiRecipeTab.getHandlerInfo(handler).getHandlerName(),
-                                        stack,
-                                        ex);
-                            }
-                        }
-                    }
+                if (fastDump) {
+                    processFastDump(writer);
+                } else {
+                    processDump(writer);
                 }
 
             } catch (Exception e) {
@@ -192,15 +171,118 @@ public class CommandRecipeId extends CommandBase {
             sendChatInfoMessage(sender, "nei.chat.recipeid.dump.finish");
         }
 
+        /**
+         * + technically more reliable and supports some weird handlers
+         * <p>
+         * - very very very slow, took 1 hour to get a dump in GTNH on M1 in background
+         * <p>
+         * - generates a lot of duplicates, up to 75% of file size is duplicates (and it's 300mb out of 400mb...)
+         */
+        private void processDump(BufferedWriter writer) throws IOException {
+            final int total = ItemList.items.size();
+            int count = 0;
+
+            for (ItemStack stack : ItemList.items) {
+                if (count % 1000 == 0) {
+                    NEIClientConfig.logger
+                            .info("({}/{}). Processing {} crafting recipes...", count, total, stack.getDisplayName());
+                }
+
+                count++;
+
+                for (ICraftingHandler handler : getCraftingHandlers(stack)) {
+                    writeLines(writer, collectRecipeLines(handler, stack.toString()));
+                }
+            }
+        }
+
+        /**
+         * - may skip some handlers if they're not TemplateRecipeHandler / just weird ones (like gendustry handlers)
+         * <p>
+         * + but in terms of GT recipes everything is dumped, and surprisingly it catches even more GT recipes than the
+         * regular dumper (in particular, it has a lot of new hammer, extruder, blast furnace, etc. recipes)
+         * <p>
+         * + way faster, took like 2.5 minutes
+         * <p>
+         * + doesn't have nearly as many duplicates, but something can still slip in, for example from
+         * FuelRecipeHandler. The dump has a size of only 100 MB instead of 400 MB from the regular dumper
+         */
+        private void processFastDump(BufferedWriter writer) throws IOException {
+            final ArrayList<ICraftingHandler> handlers = getCraftingHandlers();
+            final int total = handlers.size();
+            final AtomicInteger count = new AtomicInteger();
+
+            try {
+                final List<List<String>> linesByHandler = ItemList.forkJoinPool.submit(() -> {
+                    return handlers.parallelStream().map(handler -> {
+                        final String handlerName = GuiRecipeTab.getHandlerInfo(handler).getHandlerName();
+                        final List<String> lines = collectRecipeLines(handler, handlerName);
+                        final int current = count.incrementAndGet();
+                        NEIClientConfig.logger
+                                .info("({}/{}). Processed {} handler recipes.", current, total, handlerName);
+                        return lines;
+                    }).collect(Collectors.toList());
+                }).get();
+
+                for (List<String> lines : linesByHandler) {
+                    writeLines(writer, lines);
+                }
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                throw new IOException("Interrupted while dumping RecipeId", e);
+            } catch (ExecutionException e) {
+                throw new IOException("Error while dumping RecipeId", e);
+            }
+        }
+
+        private List<String> collectRecipeLines(ICraftingHandler handler, String context) {
+            final List<String> lines = new ArrayList<>(handler.numRecipes());
+
+            for (int index = 0, num = handler.numRecipes(); index < num; index++) {
+                try {
+                    final Recipe recipe = Recipe.of(handler, index);
+                    if (!recipe.getIngredients().isEmpty() && !recipe.getResults().isEmpty()) {
+                        lines.add(NBTJson.toJson(recipe.getRecipeId().toJsonObject()));
+                    }
+                } catch (Exception ex) {
+                    NEIClientConfig.logger.error(
+                            "Found Broken RecipeId {}:{}",
+                            GuiRecipeTab.getHandlerInfo(handler).getHandlerName(),
+                            context,
+                            ex);
+                }
+            }
+
+            return lines;
+        }
+
+        private void writeLines(BufferedWriter writer, List<String> lines) throws IOException {
+            for (String line : lines) {
+                writer.write(line);
+                writer.newLine();
+            }
+        }
+
         private ArrayList<ICraftingHandler> getCraftingHandlers(Object... results) {
             return new RecipeHandlerQuery<>(
-                    h -> h.getRecipeHandler("item", results),
+                    handler -> handler.getRecipeHandler("item", results),
                     this.craftinghandlers,
                     this.serialCraftingHandlers,
                     "Error while looking up crafting recipe",
-                    "outputId: item",
-                    "results: " + Arrays.toString(results))
-                            .runWithProfiling(NEIClientUtils.translate("recipe.concurrent.crafting"));
+                    "outputId: item").runWithProfiling(NEIClientUtils.translate("recipe.concurrent.crafting"));
+        }
+
+        private ArrayList<ICraftingHandler> getCraftingHandlers() {
+            return new RecipeHandlerQuery<>(handler -> {
+                if (handler instanceof TemplateRecipeHandler templateHandler) {
+                    return templateHandler.getAllRecipeHandler();
+                }
+                return handler.getRecipeHandler("all");
+            },
+                    this.craftinghandlers,
+                    this.serialCraftingHandlers,
+                    "Error while looking up crafting recipe",
+                    "outputId: all").runWithProfiling(NEIClientUtils.translate("recipe.concurrent.crafting"));
         }
 
     }
@@ -212,7 +294,7 @@ public class CommandRecipeId extends CommandBase {
 
     @Override
     public String getCommandUsage(ICommandSender sender) {
-        return "/recipeid dump <filename> OR /recipeid diff <prev-filename> <curr-filename> [subset name]";
+        return "/recipeid [dump|fast-dump] <filename> OR /recipeid diff <prev-filename> <curr-filename> <subset name>";
     }
 
     @Override
@@ -220,12 +302,27 @@ public class CommandRecipeId extends CommandBase {
         return 0;
     }
 
+    @Override
+    public List<String> addTabCompletionOptions(ICommandSender sender, String[] args) {
+        if (args.length == 1) {
+            return getListOfStringsMatchingLastWord(args, "dump", "fast-dump", "diff");
+        }
+
+        if ("diff".equals(args[0]) && (args.length == 2 || args.length == 3)) {
+            return getListOfStringsMatchingLastWord(args, getRecipeIdFilenames());
+        }
+
+        return Collections.emptyList();
+    }
+
     @Override
     public void processCommand(ICommandSender sender, String[] args) {
         final String command = args.length == 0 ? null : args[0];
 
         if ("dump".equals(command)) {
-            processDumpCommand(sender, args);
+            processDumpCommand(sender, args, false);
+        } else if ("fast-dump".equals(command)) {
+            processDumpCommand(sender, args, true);
         } else if ("diff".equals(command)) {
             processDiffCommand(sender, args);
         } else {
@@ -263,7 +360,7 @@ public class CommandRecipeId extends CommandBase {
         (new ProcessDiffThread(sender, prevFilename, currFilename, diffFilename)).start();
     }
 
-    protected void processDumpCommand(ICommandSender sender, String[] args) {
+    protected void processDumpCommand(ICommandSender sender, String[] args, boolean fastDump) {
 
         if (args.length > 2) {
             sendChatErrorMessage(sender, "nei.chat.recipeid.many_params", getCommandUsage(sender));
@@ -274,13 +371,30 @@ public class CommandRecipeId extends CommandBase {
         final String currFilename = args.length == 2 ? args[1] : "recipeId";
         if (!dir.exists()) dir.mkdirs();
 
-        (new ProcessDumpThread(sender, getFile(currFilename))).start();
+        (new ProcessDumpThread(sender, getFile(currFilename), fastDump)).start();
     }
 
     private static File getFile(String filename) {
         return new File(CommonUtils.getMinecraftDir(), "recipeid/" + filename + ".json");
     }
 
+    private static String[] getRecipeIdFilenames() {
+        final File dir = new File(CommonUtils.getMinecraftDir(), "recipeid");
+        final File[] files = dir.listFiles((currentDir, name) -> name.endsWith(".json"));
+
+        if (files == null || files.length == 0) {
+            return new String[] {};
+        }
+
+        final String[] names = new String[files.length];
+        for (int i = 0; i < files.length; i++) {
+            final String name = files[i].getName();
+            names[i] = name.substring(0, name.length() - ".json".length());
+        }
+
+        return names;
+    }
+
     private static void sendChatInfoMessage(ICommandSender sender, String translationKey, Object... args) {
         sender.addChatMessage(
                 new ChatComponentTranslation(translationKey, args)

src/main/java/codechicken/nei/recipe/GuiCraftingRecipe.java

@@ -21,7 +21,6 @@ import codechicken.nei.NEIClientUtils;
 import codechicken.nei.bookmark.BookmarkItem.BookmarkItemType;
 import codechicken.nei.bookmark.BookmarksGridSlot;
 import codechicken.nei.recipe.Recipe.RecipeId;
-import codechicken.nei.recipe.TemplateRecipeHandler.RecipeTransferRect;
 
 public class GuiCraftingRecipe extends GuiRecipe<ICraftingHandler> {
 
@@ -103,18 +102,7 @@ public class GuiCraftingRecipe extends GuiRecipe<ICraftingHandler> {
 
     private static ICraftingHandler buildAllRecipesHandler(ICraftingHandler handler) {
         if (handler instanceof TemplateRecipeHandler templateHandler) {
-            TemplateRecipeHandler allHandler = templateHandler.newInstance();
-            if (allHandler.transferRects.isEmpty()) {
-                allHandler.loadCraftingRecipes("all");
-                return allHandler;
-            }
-            for (RecipeTransferRect rect : allHandler.transferRects) {
-                if (allHandler.specifyTransferRect() == null
-                        || allHandler.specifyTransferRect().equals(rect.outputId)) {
-                    allHandler.loadCraftingRecipes(rect.outputId, rect.results);
-                    return allHandler;
-                }
-            }
+            return templateHandler.getAllRecipeHandler();
         }
         return handler.getRecipeHandler("all");
     }

src/main/java/codechicken/nei/recipe/TemplateRecipeHandler.java

@@ -535,6 +535,24 @@ public abstract class TemplateRecipeHandler implements ICraftingHandler, IUsageH
         }
     }
 
+    public ICraftingHandler getAllRecipeHandler() {
+        TemplateRecipeHandler handler = newInstance();
+        if (handler.transferRects.isEmpty()) {
+            handler.loadCraftingRecipes("all");
+            return handler;
+        }
+
+        final String transferRectId = handler.specifyTransferRect();
+        for (RecipeTransferRect rect : handler.transferRects) {
+            if (transferRectId == null || transferRectId.equals(rect.outputId)) {
+                handler.loadCraftingRecipes(rect.outputId, rect.results);
+                return handler;
+            }
+        }
+
+        return handler.getRecipeHandler("all");
+    }
+
     public ICraftingHandler getRecipeHandler(String outputId, Object... results) {
         TemplateRecipeHandler handler = newInstance();
         handler.loadCraftingRecipes(outputId, results);