Free chunk-meshing memory when leaving a world (#2313)

glsm/src/main/java/com/gtnewhorizons/angelica/config/SystemProperties.java

@@ -77,6 +77,8 @@ public final class SystemProperties {
     public static final Boolean FLYBY_FPS_GRAPH = parseBooleanOverride("angelica.flyby.fpsGraph");
     public static final boolean FLYBY_DEBUG_DUMP = Boolean.getBoolean("angelica.flyby.debugDump");
     public static final String FLYBY_CRASH_TEST = System.getProperty("angelica.flyby.crashTest", "");
+    public static final boolean FLYBY_HEAP_CHECK = Boolean.getBoolean("angelica.flyby.heapCheck");
+    public static final int FLYBY_VISITS = Integer.getInteger("angelica.flyby.visits", 1);
 
     // Debug
     public static final boolean LWJGL_DEBUG = Boolean.getBoolean("org.lwjgl.util.Debug");

scripts/flyby-smoke-check.py

@@ -20,7 +20,8 @@ BAD_LINE = re.compile(
     r"Debug screen entry \S+ failed and is disabled|needs the batching font renderer|"
     r"Flyby debugEntries:|Flyby: debug entry list is not available|Flyby: unknown debugProfile|"
     r"Debug screen compat entry for \S+ failed to register|"
-    r"Failed to (?:load|save|set aside the unreadable) debug screen profile|toggle listeners kept changing"
+    r"Failed to (?:load|save|set aside the unreadable) debug screen profile|toggle listeners kept changing|"
+    r"Flyby heap check: (?:FAIL|SKIPPED)"
 )
 OVERLAY_RE = re.compile(r"Flyby debug overlay shot=(\d+) mode=(\w+)(?:.* renderers=(\S*))?")
 LINE_RE = re.compile(r"Flyby debug line shot=(\d+) id=(\S+) text=(.*)")
@@ -125,7 +126,7 @@ def check(variant_dir, exit_status, new_crashes, opts=None):
         else:
             bad = [l for l in lines if BAD_LINE.search(l)]
         if bad:
-            reasons.append(str(len(bad)) + " mixin/injection/scene failure line(s)")
+            reasons.append(str(len(bad)) + " mixin/injection/scene/heap-check failure line(s)")
             details.extend(["  " + l.strip()[:200] for l in bad[:3]])
 
         offending = []
@@ -209,6 +210,7 @@ def self_test():
             ("mixin-error", GOOD_LOG + ["[10:00:05] [main/ERROR]: InvalidInjectionException: bad"], {}, 1),
             ("small-shot", GOOD_LOG, {"shot_size": 100}, 1),
             ("bad-entries", GOOD_LOG + ["[10:00:06] [Client thread/ERROR]: Flyby debugEntries: unknown entry id 'test:nope'"], {}, 1),
+            ("heap-check-fail", GOOD_LOG + ["[10:00:06] [Client thread/WARN] [Angelica/Flyby/]: Flyby heap check: FAIL WorldClient=1"], {}, 1),
         ]
         for name, log, kw, want in cases:
             d = make_variant(root, name, log, **kw)

scripts/flyby-smoke.sh

@@ -15,6 +15,7 @@
 #   SMOKE_PITCH     - camera pitch in degrees, positive looks down (default: 15).
 #   SMOKE_WARMUP_TICKS - ticks at the first pose before the pan (default: 100).
 #   SMOKE_ARGS      - extra -D flags for the client, word-split (default: none).
+#                     World-leak check: "-Dangelica.flyby.visits=3 -Dangelica.flyby.heapCheck=true" (FAIL/SKIPPED fails the variant).
 #   SMOKE_CHECK_ARGS - extra flyby-smoke-check.py flags, word-split (default: none).
 #   SMOKE_FULLSCREEN - 0 keeps options.txt's window mode (default: 1, fullscreen for the run).
 #

src/main/java/com/gtnewhorizons/angelica/debug/flyby/FlybyHeapCheck.java

@@ -0,0 +1,133 @@
+package com.gtnewhorizons.angelica.debug.flyby;
+
+import com.gtnewhorizons.angelica.config.SystemProperties;
+import com.gtnewhorizons.angelica.debug.profiling.AsprofRecorder;
+import com.gtnewhorizons.angelica.rendering.celeritas.AngelicaRenderSectionManager;
+import com.gtnewhorizons.angelica.rendering.celeritas.DeferredMeshScheduler;
+import com.gtnewhorizons.angelica.rendering.celeritas.world.WorldSlice;
+import com.gtnewhorizons.angelica.rendering.celeritas.world.cloned.ClonedChunkSection;
+import com.sun.management.HotSpotDiagnosticMXBean;
+import it.unimi.dsi.fastutil.objects.Object2LongMap;
+import it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap;
+import net.minecraft.client.entity.EntityClientPlayerMP;
+import net.minecraft.client.multiplayer.WorldClient;
+import net.minecraft.world.chunk.Chunk;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.embeddedt.embeddium.impl.render.chunk.RenderSection;
+import org.embeddedt.embeddium.impl.render.chunk.compile.ChunkBuildBuffers;
+
+import javax.management.ObjectName;
+import java.io.File;
+import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
+import java.util.List;
+
+final class FlybyHeapCheck {
+    private static final Logger LOGGER = LogManager.getLogger("Angelica/Flyby");
+    private static final int TOP_GROWERS = 10;
+    private static final int MIN_REPORTS_FOR_GROWTH = 3;
+
+    private final List<Object2LongOpenHashMap<String>> history = new ArrayList<>();
+    private final Object2LongOpenHashMap<String> maxRetained = new Object2LongOpenHashMap<>();
+    private String skipReason;
+
+    private static Class<?>[] mustBeZero() {
+        return new Class<?>[] {AngelicaRenderSectionManager.class, WorldSlice.class, ClonedChunkSection.class, DeferredMeshScheduler.class, RenderSection.class, ChunkBuildBuffers.class, WorldClient.class};
+    }
+
+    private static Class<?>[] watched() {
+        return new Class<?>[] {Chunk.class, EntityClientPlayerMP.class};
+    }
+
+    void report(String label) {
+        final Object2LongOpenHashMap<String> counts;
+        try {
+            // Objects with a pending finalize() (AE2 part monitors) count as live and keep their world until finalized.
+            System.gc();
+            System.runFinalization();
+            counts = liveHistogram();
+        } catch (Exception e) {
+            LOGGER.warn("Flyby heap check: histogram failed", e);
+            skip("histogram failed: " + e);
+            return;
+        }
+        final StringBuilder sb = new StringBuilder("Flyby heap ").append(label).append(':');
+        for (Class<?> type : mustBeZero()) {
+            final long count = counts.getLong(type.getName());
+            if (count > maxRetained.getLong(type.getName())) maxRetained.put(type.getName(), count);
+            sb.append(' ').append(type.getSimpleName()).append('=').append(count);
+        }
+        sb.append(" | watched");
+        for (Class<?> type : watched()) {
+            sb.append(' ').append(type.getSimpleName()).append('=').append(counts.getLong(type.getName()));
+        }
+        LOGGER.info(sb);
+        history.add(counts);
+    }
+
+    void skip(String reason) {
+        if (skipReason == null) skipReason = reason;
+    }
+
+    void finish() {
+        if (skipReason != null) {
+            LOGGER.warn("Flyby heap check: SKIPPED ({})", skipReason);
+            return;
+        }
+        if (history.size() >= MIN_REPORTS_FOR_GROWTH) logGrewEveryVisit();
+        final StringBuilder retained = new StringBuilder();
+        for (Class<?> type : mustBeZero()) {
+            final long max = maxRetained.getLong(type.getName());
+            if (max > 0) retained.append(' ').append(type.getSimpleName()).append('=').append(max);
+        }
+        if (retained.length() == 0) {
+            LOGGER.info("Flyby heap check: PASS");
+            return;
+        }
+        LOGGER.warn("Flyby heap check: FAIL{}", retained);
+        dump();
+    }
+
+    private void logGrewEveryVisit() {
+        final Object2LongOpenHashMap<String> first = history.get(0);
+        final Object2LongOpenHashMap<String> last = history.get(history.size() - 1);
+        final List<Object2LongMap.Entry<String>> grew = new ArrayList<>();
+        for (Object2LongMap.Entry<String> e : last.object2LongEntrySet()) {
+            if (roseAtEveryReport(e.getKey())) grew.add(e);
+        }
+        grew.sort((a, b) -> Long.compare(b.getLongValue() - first.getLong(b.getKey()), a.getLongValue() - first.getLong(a.getKey())));
+        for (int i = 0; i < Math.min(TOP_GROWERS, grew.size()); i++) {
+            final Object2LongMap.Entry<String> e = grew.get(i);
+            LOGGER.info("Flyby heap: grew at every visit: {} {} -> {}", e.getKey(), first.getLong(e.getKey()), e.getLongValue());
+        }
+    }
+
+    private boolean roseAtEveryReport(String className) {
+        for (int i = 1; i < history.size(); i++) {
+            if (history.get(i).getLong(className) <= history.get(i - 1).getLong(className)) return false;
+        }
+        return true;
+    }
+
+    private static void dump() {
+        final File file = new File(SystemProperties.PROFILE_DIR, AsprofRecorder.fileName("leak", "hprof", System.currentTimeMillis())).getAbsoluteFile();
+        try {
+            file.getParentFile().mkdirs();
+            ManagementFactory.getPlatformMXBean(HotSpotDiagnosticMXBean.class).dumpHeap(file.getPath(), true);
+            LOGGER.info("Flyby heap: dumped {}", file.getPath());
+        } catch (Exception e) {
+            LOGGER.warn("Flyby heap dump to {} failed", file.getPath(), e);
+        }
+    }
+
+    private static Object2LongOpenHashMap<String> liveHistogram() throws Exception {
+        final String histogram = (String) ManagementFactory.getPlatformMBeanServer().invoke(new ObjectName("com.sun.management:type=DiagnosticCommand"), "gcClassHistogram", new Object[] {null}, new String[] {String[].class.getName()});
+        final Object2LongOpenHashMap<String> counts = new Object2LongOpenHashMap<>();
+        for (String line : histogram.split("\n")) {
+            final String[] cols = line.trim().split("\\s+");
+            if (cols.length >= 4 && cols[0].endsWith(":")) counts.put(cols[3], Long.parseLong(cols[1]));
+        }
+        return counts;
+    }
+}

src/main/java/com/gtnewhorizons/angelica/debug/flyby/FlybyRunner.java

@@ -13,15 +13,19 @@ import com.gtnewhorizons.angelica.glsm.profiling.DebugCounters;
 import com.gtnewhorizons.angelica.rendering.FpsReducer;
 import com.gtnewhorizons.angelica.rendering.FramePacer;
 import com.gtnewhorizons.angelica.rendering.IsbrhTestRenderer;
+import com.gtnewhorizons.angelica.rendering.celeritas.CeleritasWorldRenderer;
 import com.sun.management.OperatingSystemMXBean;
 import com.sun.management.ThreadMXBean;
+import cpw.mods.fml.client.FMLClientHandler;
 import cpw.mods.fml.common.FMLCommonHandler;
 import cpw.mods.fml.common.eventhandler.SubscribeEvent;
 import cpw.mods.fml.common.gameevent.TickEvent;
 import net.minecraft.client.Minecraft;
 import net.minecraft.client.entity.EntityClientPlayerMP;
 import net.minecraft.client.gui.GuiMainMenu;
+import net.minecraft.client.gui.GuiSelectWorld;
 import net.minecraft.entity.player.EntityPlayerMP;
+import net.minecraft.server.integrated.IntegratedServer;
 import net.minecraft.server.MinecraftServer;
 import net.minecraft.util.ChatComponentText;
 import net.minecraft.util.ChunkCoordinates;
@@ -60,12 +64,17 @@ public final class FlybyRunner {
 
     public static final FlybyRunner INSTANCE = new FlybyRunner();
 
-    private enum State { IDLE, WAITING, PREPARING, WARMUP, RUNNING, SETTLE, EXITING, DONE }
+    private enum State { IDLE, WAITING, PREPARING, WARMUP, RUNNING, SETTLE, EXITING, MENU, DONE }
 
     private static final int EXIT_TICKS = 20;
     private static final int SETTLE_TICKS = 40;
+    private static final int MENU_SETTLE_TICKS = 20;
 
+    private final FlybyHeapCheck heapCheck = new FlybyHeapCheck();
     private State state = State.IDLE;
+    private int visit;
+    private String rejoinFolder;
+    private String rejoinName;
     private FlybyRoute route;
     private int warmupTicks;
     private int runLength;
@@ -189,6 +198,14 @@ public final class FlybyRunner {
         this.waitForFocus = SystemProperties.FLYBY_WAIT_FOR_FOCUS;
         this.exitWhenDone = SystemProperties.FLYBY_EXIT_WHEN_DONE;
         this.jfr = SystemProperties.FLYBY_JFR;
+        if (this.jfr && SystemProperties.FLYBY_VISITS > 1) {
+            LOGGER.error("Flyby: angelica.flyby.jfr records one file per run and cannot cover {} visits; exiting instead of profiling only the first", SystemProperties.FLYBY_VISITS);
+            FMLCommonHandler.instance().exitJava(1, false);
+        }
+        if (!this.exitWhenDone && (SystemProperties.FLYBY_VISITS > 1 || SystemProperties.FLYBY_HEAP_CHECK)) {
+            LOGGER.error("Flyby: angelica.flyby.visits and angelica.flyby.heapCheck leave and rejoin the world, which needs angelica.flyby.exitWhenDone=true; exiting instead of ignoring them");
+            FMLCommonHandler.instance().exitJava(1, false);
+        }
         if (!SystemProperties.FLYBY_COMMANDS.isEmpty()) {
             this.sceneCommands = FlybyScene.load(SystemProperties.FLYBY_COMMANDS);
             LOGGER.info("Flyby scene '{}': {} commands", SystemProperties.FLYBY_COMMANDS, this.sceneCommands.length);
@@ -251,7 +268,16 @@ public final class FlybyRunner {
         if (this.state == State.EXITING) {
             if (++this.tick >= EXIT_TICKS) {
                 this.state = State.DONE;
-                this.exitGame(mc);
+                this.leave(mc);
+            }
+            return;
+        }
+
+        if (this.state == State.MENU) {
+            if (++this.tick >= MENU_SETTLE_TICKS) {
+                this.state = State.DONE;
+                this.checkHeapAtMenu();
+                this.afterMenu(mc);
             }
             return;
         }
@@ -892,14 +918,51 @@ public final class FlybyRunner {
         this.state = this.exitWhenDone ? State.EXITING : State.DONE;
     }
 
-    private void exitGame(Minecraft mc) {
+    private void leave(Minecraft mc) {
         this.stopRecording(mc);
-        LOGGER.info("Flyby complete, shutting down.");
+        this.rejoinFolder = null;
+        this.rejoinName = null;
+        if (++this.visit < SystemProperties.FLYBY_VISITS) {
+            final IntegratedServer server = mc.getIntegratedServer();
+            if (server != null) {
+                this.rejoinFolder = server.getFolderName();
+                this.rejoinName = server.getWorldName();
+                LOGGER.info("Flyby: visit {} of {} done, rejoining '{}'", this.visit, SystemProperties.FLYBY_VISITS, this.rejoinFolder);
+            } else {
+                LOGGER.error("Flyby: no singleplayer world to revisit, shutting down");
+            }
+        }
+        if (this.rejoinFolder == null) LOGGER.info("Flyby complete, shutting down.");
         if (mc.theWorld != null) {
             mc.theWorld.sendQuittingDisconnectingPacket();
             mc.loadWorld(null);
+            worldChangesDiscarded = false;
             mc.displayGuiScreen(new GuiMainMenu());
         }
+        if (SystemProperties.FLYBY_HEAP_CHECK) {
+            this.state = State.MENU;
+            this.tick = 0;
+        } else {
+            this.afterMenu(mc);
+        }
+    }
+
+    private void checkHeapAtMenu() {
+        final CeleritasWorldRenderer renderer = CeleritasWorldRenderer.getInstanceOrNull();
+        if (renderer != null && renderer.isActive()) this.heapCheck.skip("renderer still loaded: no unload hook");
+        else this.heapCheck.report("after visit " + this.visit);
+    }
+
+    private void afterMenu(Minecraft mc) {
+        if (this.rejoinFolder != null) {
+            FMLClientHandler.instance().tryLoadExistingWorld(new GuiSelectWorld(new GuiMainMenu()), this.rejoinFolder, this.rejoinName);
+            if (mc.getIntegratedServer() != null) {
+                this.startFromProperties();
+                return;
+            }
+            LOGGER.error("Flyby: could not rejoin '{}', shutting down", this.rejoinFolder);
+        }
+        if (SystemProperties.FLYBY_HEAP_CHECK) this.heapCheck.finish();
         mc.shutdown();
     }
 

src/main/java/com/gtnewhorizons/angelica/debug/profiling/AsprofRecorder.java

@@ -121,7 +121,7 @@ public final class AsprofRecorder {
         return opts.isEmpty() ? "start,jfr,file=" + file : "start,jfr," + opts + ",file=" + file;
     }
 
-    static String fileName(String tag, String ext, long nowMillis) {
+    public static String fileName(String tag, String ext, long nowMillis) {
         final SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd-HHmmss", Locale.ROOT);
         return "angelica-" + tag + "-" + format.format(new Date(nowMillis)) + "." + ext;
     }

src/main/java/com/gtnewhorizons/angelica/rendering/AngelicaRenderQueue.java

@@ -57,6 +57,10 @@ public class AngelicaRenderQueue {
         LockSupport.unpark(GLStateManager.getMainThread());
     }
 
+    public static void remove(Runnable runnable) {
+        TASKS.removeIf(t -> t == runnable);
+    }
+
     public static int processTasks(int max) {
         int tasksRun = 0;
         while(tasksRun < max) {

src/main/java/com/gtnewhorizons/angelica/rendering/celeritas/AngelicaRenderSectionManager.java

@@ -56,6 +56,7 @@ public class AngelicaRenderSectionManager extends RenderSectionManager {
     private static final long P_MESH_DEVICE_ALLOCATED = Tracy.plotHandle("mesh.deviceAllocated", TracyBackend.PLOT_FORMAT_MEMORY);
     private static final long P_MESH_DEFERRED_PENDING = Tracy.plotHandle("mesh.deferredPending");
     private static final long P_MESH_DEFERRED_FINISHING = Tracy.plotHandle("mesh.deferredFinishing");
+    private static final long P_MESH_DEFERRED_FREE_SLICES = Tracy.plotHandle("mesh.deferredFreeSlices");
 
     private static final Tracy.ZoneId Z_BIOME_REBUILDS = Tracy.zoneId("biomeRebuilds", Tracy.COLOR_TERRAIN);
     private static final Tracy.ZoneId Z_GRAPH_SEARCH = Tracy.zoneId("graphSearch", Tracy.COLOR_TERRAIN);
@@ -73,7 +74,7 @@ public class AngelicaRenderSectionManager extends RenderSectionManager {
         this.world = world;
         this.sectionCache = new ClonedChunkSectionCache(world);
         this.taskProvider = taskProvider;
-        this.deferredScheduler = new DeferredMeshScheduler(configuration, getBuilder(), () -> new WorldSlice(world), AngelicaRenderQueue::submit, ((RenderSectionManagerAccessor) this).angelica$getBuildResults());
+        this.deferredScheduler = new DeferredMeshScheduler(configuration, getBuilder(), () -> new WorldSlice(world), AngelicaRenderQueue::submit, AngelicaRenderQueue::remove, ((RenderSectionManagerAccessor) this).angelica$getBuildResults());
     }
 
     public static AngelicaRenderSectionManager create(ChunkVertexType vertexType, WorldClient world, int renderDistance, CommandList commandList) {
@@ -211,6 +212,7 @@ public class AngelicaRenderSectionManager extends RenderSectionManager {
     @Override
     public void updateChunks(boolean updateImmediately) {
         this.sectionCache.cleanup();
+        this.deferredScheduler.trimPools();
         this.flushBiomeRebuilds();
         if (Tracy.ENABLED) Tracy.beginZone(Z_GRAPH_SEARCH);
         try {
@@ -353,5 +355,6 @@ public class AngelicaRenderSectionManager extends RenderSectionManager {
         Tracy.plotInt(P_DRAW_REGION_BUFFERS, mem.bufferCount);
         Tracy.plotInt(P_MESH_DEFERRED_PENDING, this.deferredScheduler.pendingCount());
         Tracy.plotInt(P_MESH_DEFERRED_FINISHING, this.deferredScheduler.finishingCount());
+        Tracy.plotInt(P_MESH_DEFERRED_FREE_SLICES, this.deferredScheduler.freeSliceCount());
     }
 }

src/main/java/com/gtnewhorizons/angelica/rendering/celeritas/DeferredMeshScheduler.java

@@ -28,9 +28,12 @@ public class DeferredMeshScheduler {
     private static final long FINISH_PRIORITY = -1L;
 
     private static final Comparator<DeferredSectionMesh> NEAREST_FIRST = Comparator.comparingLong(m -> m.priority);
+    private static final int CAPTURES_PER_WORKER = 2;
+    private static final int MIN_FREE_PER_WORKER = 2;
 
     private final ChunkBuilder builder;
     private final Consumer<Runnable> mainThreadQueue;
+    private final Consumer<Runnable> mainThreadUnqueue;
     private final Queue<ChunkJobResult<? extends ChunkTaskOutput>> results;
     private final Consumer<ChunkJobResult<? extends ChunkTaskOutput>> resultSink;
     private final ObjectPool<ChunkBuildBuffers> buffers;
@@ -41,17 +44,21 @@ public class DeferredMeshScheduler {
     private final Set<DeferredSectionMesh> inFlight = ConcurrentHashMap.newKeySet();
     private final Runnable runNext = this::runNext;
     private final AtomicInteger pending = new AtomicInteger();
+    private final int minFree;
 
     private boolean destroyed;
 
-    public DeferredMeshScheduler(RenderPassConfiguration<?> config, ChunkBuilder builder, Supplier<WorldSlice> sliceFactory, Consumer<Runnable> mainThreadQueue, Queue<ChunkJobResult<? extends ChunkTaskOutput>> results) {
+    public DeferredMeshScheduler(RenderPassConfiguration<?> config, ChunkBuilder builder, Supplier<WorldSlice> sliceFactory, Consumer<Runnable> mainThreadQueue, Consumer<Runnable> mainThreadUnqueue, Queue<ChunkJobResult<? extends ChunkTaskOutput>> results) {
         this.builder = builder;
         this.mainThreadQueue = mainThreadQueue;
+        this.mainThreadUnqueue = mainThreadUnqueue;
         this.results = results;
         this.resultSink = results::add;
+        final int workers = Math.max(1, builder.getTotalThreadCount());
+        this.minFree = workers * MIN_FREE_PER_WORKER;
         this.buffers = new ObjectPool<>(() -> new ChunkBuildBuffers(config), Integer.MAX_VALUE);
         this.slices = new ObjectPool<>(sliceFactory, Integer.MAX_VALUE);
-        this.captures = new ObjectPool<>(DeferredCapture::new, Math.max(1, builder.getTotalThreadCount()) * 2);
+        this.captures = new ObjectPool<>(DeferredCapture::new, workers * CAPTURES_PER_WORKER);
     }
 
     public ChunkBuildBuffers acquireBuffers() {
@@ -93,6 +100,7 @@ public class DeferredMeshScheduler {
 
     /** Call after the builder shut down: its queued jobs were dropped without reaching their consumers. */
     public void releaseOrphans() {
+        mainThreadUnqueue.accept(runNext);
         DeferredSectionMesh m;
         while ((m = important.poll()) != null) {
             release(m);
@@ -113,6 +121,16 @@ public class DeferredMeshScheduler {
         return inFlight.size();
     }
 
+    public int freeSliceCount() {
+        return slices.freeCount();
+    }
+
+    public void trimPools() {
+        final long now = System.nanoTime();
+        slices.trim(now, minFree);
+        buffers.trim(now, minFree);
+    }
+
     void finished(DeferredSectionMesh m, @Nullable ChunkJobResult<ChunkBuildOutput> result, @Nullable ChunkJobCollector collector) {
         final long nanos = m.workerNanos + m.mainNanos;
         ChunkJobResult<ChunkBuildOutput> out = result instanceof ChunkJobResult.Success<ChunkBuildOutput> success ? new ChunkJobResult.Success<>(success.output(), nanos + success.executionTimeNanos()) : result;

src/main/java/com/gtnewhorizons/angelica/rendering/celeritas/ObjectPool.java

@@ -1,21 +1,31 @@
 package com.gtnewhorizons.angelica.rendering.celeritas;
 
 import java.util.concurrent.ConcurrentLinkedDeque;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
 
 final class ObjectPool<T> {
+    static final long TRIM_WINDOW_NANOS = TimeUnit.SECONDS.toNanos(30);
+
     private final ConcurrentLinkedDeque<T> free = new ConcurrentLinkedDeque<>();
     private final AtomicInteger freeCount = new AtomicInteger();
+    private final AtomicInteger inUse = new AtomicInteger();
     private final Supplier<T> factory;
     private final int maxFree;
 
+    private boolean started;
+    private long windowStart;
+    private int windowPeak;
+    private int previousPeak;
+
     ObjectPool(Supplier<T> factory, int maxFree) {
         this.factory = factory;
         this.maxFree = maxFree;
     }
 
     T acquire() {
+        inUse.incrementAndGet();
         final T t = free.pollFirst();
         if (t == null) return factory.get();
         freeCount.decrementAndGet();
@@ -23,10 +33,34 @@ final class ObjectPool<T> {
     }
 
     void release(T t) {
+        inUse.decrementAndGet();
         if (freeCount.incrementAndGet() > maxFree) {
             freeCount.decrementAndGet();
             return;
         }
         free.offerFirst(t);
     }
+
+    int freeCount() {
+        return freeCount.get();
+    }
+
+    void trim(long nowNanos, int minFree) {
+        final int used = inUse.get();
+        if (used > windowPeak) windowPeak = used;
+        if (!started) {
+            started = true;
+            windowStart = nowNanos;
+            return;
+        }
+        if (nowNanos - windowStart < TRIM_WINDOW_NANOS) return;
+        final int peak = Math.max(windowPeak, previousPeak);
+        previousPeak = windowPeak;
+        windowPeak = used;
+        windowStart = nowNanos;
+        final int excess = freeCount.get() - Math.max(minFree, peak + (peak >> 1) - used);
+        for (int i = (excess + 1) >> 1; i > 0 && free.pollLast() != null; i--) {
+            freeCount.decrementAndGet();
+        }
+    }
 }

src/test/java/com/gtnewhorizons/angelica/rendering/celeritas/DeferredMeshSchedulerTest.java

@@ -83,7 +83,7 @@ class DeferredMeshSchedulerTest {
             scheduler = new DeferredMeshScheduler(null, builder, () -> {
                 slicesCreated.incrementAndGet();
                 return Mockito.mock(WorldSlice.class);
-            }, queue::add, results);
+            }, queue::add, r -> queue.removeIf(t -> t == r), results);
         }
 
         DeferredSectionMesh submit(boolean important, long priority) {
@@ -213,6 +213,7 @@ class DeferredMeshSchedulerTest {
         assertEquals(0, h.scheduler.pendingCount());
         assertEquals(0, h.scheduler.finishingCount());
         assertEquals(0, h.results.size());
+        assertTrue(h.queue.isEmpty(), "queued runNext keeps the destroyed scheduler and its pools reachable");
         Mockito.verify(queued.slice, Mockito.times(1)).releaseContext();
         Mockito.verify(inFlight.slice, Mockito.times(1)).releaseContext();
         assertNull(inFlight.capture);

src/test/java/com/gtnewhorizons/angelica/rendering/celeritas/ObjectPoolTest.java

@@ -59,4 +59,27 @@ class ObjectPoolTest {
         pool.acquire();
         assertEquals(4, calls.get());
     }
+
+    @Test
+    void trimKeepsRecentPeakThenHalvesExcess() {
+        final long w = ObjectPool.TRIM_WINDOW_NANOS;
+        final int floor = 4;
+        ObjectPool<Object> pool = new ObjectPool<>(Object::new, Integer.MAX_VALUE);
+
+        Object[] burst = new Object[100];
+        for (int i = 0; i < burst.length; i++) burst[i] = pool.acquire();
+        pool.trim(0, floor);
+        for (int i = 1; i < burst.length; i++) pool.release(burst[i]);
+
+        for (long t : new long[] {w / 2, w, 2 * w}) {
+            pool.trim(t, floor);
+            assertEquals(99, pool.freeCount(), "peak from the last two windows still holds at t=" + t);
+        }
+
+        int[] expected = {51, 27, 15, 9, 6, 5, 4, 4};
+        for (int i = 0; i < expected.length; i++) {
+            pool.trim((3 + i) * w, floor);
+            assertEquals(expected[i], pool.freeCount(), "window " + (3 + i));
+        }
+    }
 }

tracy-client/src/main/java/com/gtnewhorizons/angelica/tracy/TracyClientBackend.java

@@ -18,8 +18,10 @@ import org.lwjgl.system.libffi.FFICIF;
 import org.lwjgl.system.libffi.FFIType;
 import org.lwjgl.system.libffi.LibFFI;
 
+import java.lang.ref.WeakReference;
 import java.nio.ByteBuffer;
 import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.atomic.AtomicReference;
 
 import static org.lwjgl.system.MemoryStack.stackPush;
 import static org.lwjgl.system.MemoryUtil.NULL;
@@ -138,11 +140,13 @@ public final class TracyClientBackend implements TracyBackend {
     private static final class SlotArena {
         private final long base;
         private final int strideShift;
+        private final AtomicReference<WeakReference<Thread>> owner;
         private int depth;
 
-        SlotArena(int stride) {
+        SlotArena(int stride, Thread owner) {
             this.strideShift = Integer.numberOfTrailingZeros(stride);
             this.base = nmemAllocChecked((long) stride * ZoneStack.MAX_DEPTH);
+            this.owner = new AtomicReference<>(new WeakReference<>(owner));
         }
 
         long top() {
@@ -155,11 +159,23 @@ public final class TracyClientBackend implements TracyBackend {
         }
     }
 
-    private final ThreadLocal<SlotArena> slots = ThreadLocal.withInitial(() -> {
-        final SlotArena arena = new SlotArena(ctxSize);
+    private final ThreadLocal<SlotArena> slots = ThreadLocal.withInitial(this::claimArena);
+
+    /** Arenas outlive their threads (chunk workers are recreated per world), so reuse a dead thread's arena; owners are weak because a dead thread still holds its Runnable and through it the old world. */
+    private SlotArena claimArena() {
+        final Thread self = Thread.currentThread();
+        for (SlotArena arena : ARENAS) {
+            final WeakReference<Thread> ref = arena.owner.get();
+            final Thread owner = ref.get();
+            if ((owner == null || !owner.isAlive()) && arena.owner.compareAndSet(ref, new WeakReference<>(self))) {
+                arena.depth = 0;
+                return arena;
+            }
+        }
+        final SlotArena arena = new SlotArena(ctxSize, self);
         ARENAS.add(arena);
         return arena;
-    });
+    }
 
     @Override
     public long beginZone(long srcLoc) {