From 37afddab5d6546313a10bc862c3e72d7ea46d034 Mon Sep 17 00:00:00 2001 From: Jason Mitchell Date: Fri, 2 Oct 2026 22:35:08 -0700 Subject: [PATCH] Cache transformed and compiled shaders on disk (#2194) --- .gitignore | 1 + glsm/build.gradle.kts | 1 + .../glsm/CompatShaderTransformer.java | 29 +- .../angelica/glsm/shader/ShaderCacheIO.java | 106 ++++++ .../angelica/glsm/shader/ShaderDiskCache.java | 307 +++++++++++++++++ .../angelica/glsm/shader/SpirvCompiler.java | 48 ++- .../glsm/CompatShaderTransformerTest.java | 29 ++ .../glsm/shader/ShaderDiskCacheTest.java | 125 +++++++ .../shader/SpirvCompilerDiskCacheTest.java | 61 ++++ .../angelica/sdlgpu/shader/ClipZRemap.java | 16 - .../sdlgpu/shader/PerFrameBlockInjector.java | 17 - .../sdlgpu/shader/SamplerStripper.java | 15 - .../angelica/sdlgpu/shader/ShaderManager.java | 266 ++++++++------ .../sdlgpu/shader/ShaderStageSerializer.java | 197 +++++++++++ .../shader/cross/CrossCompileCache.java | 81 +++++ .../sdlgpu/shader/cross/CrossCompileUtil.java | 9 + .../sdlgpu/shader/dxbc/D3DCompiler.java | 3 + .../sdlgpu/shader/dxbc/DxbcCrossCompile.java | 66 +--- .../sdlgpu/shader/msl/MslCrossCompile.java | 70 +--- .../shader/FontFilterVertexSamplerTest.java | 4 +- .../shader/PerFrameBlockInjectorTest.java | 36 +- .../PerFrameBlockLocationRoutingTest.java | 2 +- .../shader/PerFrameBlockReadOnlyTest.java | 2 +- .../PrewarmSingleParseEquivalenceTest.java | 27 +- .../PrewarmTreeHandoffEquivalenceTest.java | 4 +- .../SdlSamplerStripperPreprocessorTest.java | 16 +- .../ShaderSourceTransformCacheTest.java | 10 +- .../shader/ShaderStageSerializerTest.java | 207 +++++++++++ .../sdlgpu/shader/ShaderTransformChain.java | 39 ++- .../sdlgpu/shader/StageDiskCacheTest.java | 324 ++++++++++++++++++ .../sdlgpu/shader/dxbc/DxbcCompileTest.java | 47 ++- .../msl/FragmentStorageTextureSlotTest.java | 5 +- .../shader/msl/MslCrossCompileTest.java | 55 ++- .../angelica/proxy/ClientProxy.java | 8 + .../transform/RwImageStoreExtractor.java | 4 + .../pipeline/transform/ShaderTransformer.java | 68 +++- .../parameter/AttributeParameters.java | 7 + .../transform/parameter/Parameters.java | 18 + .../parameter/TextureStageParameters.java | 7 + .../ShaderTransformerDiskCacheTest.java | 132 +++++++ 40 files changed, 2131 insertions(+), 338 deletions(-) create mode 100644 glsm/src/main/java/com/gtnewhorizons/angelica/glsm/shader/ShaderCacheIO.java create mode 100644 glsm/src/main/java/com/gtnewhorizons/angelica/glsm/shader/ShaderDiskCache.java create mode 100644 glsm/src/test/java/com/gtnewhorizons/angelica/glsm/shader/ShaderDiskCacheTest.java create mode 100644 glsm/src/test/java/com/gtnewhorizons/angelica/glsm/shader/SpirvCompilerDiskCacheTest.java create mode 100644 sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderStageSerializer.java create mode 100644 sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/cross/CrossCompileCache.java create mode 100644 sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderStageSerializerTest.java create mode 100644 sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/StageDiskCacheTest.java create mode 100644 src/test/java/net/coderbot/iris/pipeline/transform/ShaderTransformerDiskCacheTest.java diff --git a/.gitignore b/.gitignore index 3e6696eeb..a2b214533 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,4 @@ dist/tracy-natives/ *.jfr tracy-client/angelica/ +shaderc_failures/ diff --git a/glsm/build.gradle.kts b/glsm/build.gradle.kts index 4373c5227..553954b73 100644 --- a/glsm/build.gradle.kts +++ b/glsm/build.gradle.kts @@ -320,6 +320,7 @@ val spirvTest by tasks.registering(Test::class) { filter { includeTestsMatching("com.gtnewhorizons.angelica.glsm.shader.SpirvShaderTranslator*Test") includeTestsMatching("com.gtnewhorizons.angelica.glsm.ffp.FFPUniformBlockSpirvLayoutTest") + includeTestsMatching("com.gtnewhorizons.angelica.glsm.shader.SpirvCompilerDiskCacheTest") } // libshaderc.so / libspirv-cross.so are extracted by extractNatives3. diff --git a/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/CompatShaderTransformer.java b/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/CompatShaderTransformer.java index e2bf1d0a1..b5ac0bb29 100644 --- a/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/CompatShaderTransformer.java +++ b/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/CompatShaderTransformer.java @@ -3,6 +3,7 @@ package com.gtnewhorizons.angelica.glsm; import com.gtnewhorizon.gtnhlib.client.renderer.vertex.VertexFormatElement.Usage; import com.gtnewhorizons.angelica.config.SystemProperties; import com.gtnewhorizons.angelica.glsm.backend.RenderBackend; +import com.gtnewhorizons.angelica.glsm.shader.ShaderDiskCache; import com.gtnewhorizons.angelica.glsm.shader.SpirvShaderTranslator; import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.runtime.tree.TerminalNode; @@ -139,12 +140,20 @@ public class CompatShaderTransformer { if (cached != null) { result = cached; } else { - try { - result = transformInternal(source, isFragment); + final ShaderDiskCache.Key dk = ShaderDiskCache.isEnabled() ? diskKey(source, isFragment) : null; + final String disk = dk != null ? ShaderDiskCache.getString(dk) : null; + if (disk != null) { + result = disk; cache.put(key, result); - } catch (Exception e) { - GLStateManager.LOGGER.warn("CompatShaderTransformer: AST transformation failed, falling back to version fixup only", e); - result = fixupVersion(source); + } else { + try { + result = transformInternal(source, isFragment); + cache.put(key, result); + if (dk != null) ShaderDiskCache.putString(dk, result); + } catch (Exception e) { + GLStateManager.LOGGER.warn("CompatShaderTransformer: AST transformation failed, falling back to version fixup only", e); + result = fixupVersion(source); + } } } } @@ -164,6 +173,14 @@ public class CompatShaderTransformer { return result; } + private static int minGlslVersion() { + return RENDER_BACKEND != null ? RENDER_BACKEND.getMinGLSLVersion() : 330; + } + + static ShaderDiskCache.Key diskKey(String source, boolean isFragment) { + return ShaderDiskCache.key("compat").str(source).b(isFragment).i(minGlslVersion()); + } + public static boolean isCoreShader(String source) { final Matcher vm = VERSION_PATTERN.matcher(source); if (vm.find()) { @@ -186,7 +203,7 @@ public class CompatShaderTransformer { declaredVersion = Integer.parseInt(versionMatcher.group(1)); } - final int targetVersion = Math.max(declaredVersion, RENDER_BACKEND != null ? RENDER_BACKEND.getMinGLSLVersion() : 330); + final int targetVersion = Math.max(declaredVersion, minGlslVersion()); // Pre-parse reserved word renaming - prevents ANTLR parse failures source = GlslTransformUtils.replaceTexture(source); diff --git a/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/shader/ShaderCacheIO.java b/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/shader/ShaderCacheIO.java new file mode 100644 index 000000000..5b81d6923 --- /dev/null +++ b/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/shader/ShaderCacheIO.java @@ -0,0 +1,106 @@ +package com.gtnewhorizons.angelica.glsm.shader; + +import it.unimi.dsi.fastutil.objects.Object2ObjectLinkedOpenHashMap; +import me.eigenraven.lwjgl3ify.api.Lwjgl3Aware; +import org.lwjgl.Version; +import org.lwjgl.system.MemoryUtil; +import org.lwjgl.system.SharedLibrary; + +import java.io.DataOutputStream; +import java.io.File; +import java.io.IOException; +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Set; +import java.util.TreeSet; + +@Lwjgl3Aware +public final class ShaderCacheIO { + + private ShaderCacheIO() {} + + public static byte[] toHeap(ByteBuffer buf) { + final byte[] heap = new byte[buf.remaining()]; + buf.duplicate().get(heap); + return heap; + } + + public static ByteBuffer toNative(byte[] data) { + final ByteBuffer buf = MemoryUtil.memAlloc(data.length); + buf.put(data).flip(); + return buf; + } + + public static void lruPut(Object2ObjectLinkedOpenHashMap map, K key, V value, int max) { + synchronized (map) { + map.putAndMoveToFirst(key, value); + while (map.size() > max) map.removeLast(); + } + } + + public static void writeStr(DataOutputStream out, String s) throws IOException { + if (s == null) { + out.writeInt(-1); + return; + } + final byte[] utf8 = s.getBytes(StandardCharsets.UTF_8); + out.writeInt(utf8.length); + out.write(utf8); + } + + public static String readStr(ByteBuffer in) { + final int n = in.getInt(); + if (n == -1) return null; + final int pos = in.position(); + in.position(pos + n); + return new String(in.array(), in.arrayOffset() + pos, n, StandardCharsets.UTF_8); + } + + public static int readCount(ByteBuffer in) { + final int n = in.getInt(); + if (n < 0 || n > in.remaining()) throw new BufferUnderflowException(); + return n; + } + + public static void writeBytes(DataOutputStream out, byte[] data) throws IOException { + out.writeInt(data.length); + out.write(data); + } + + public static byte[] readBytes(ByteBuffer in) { + final byte[] out = new byte[readCount(in)]; + in.get(out); + return out; + } + + public static void writeInts(DataOutputStream out, int[] values) throws IOException { + out.writeInt(values.length); + for (int v : values) out.writeInt(v); + } + + public static int[] readInts(ByteBuffer in) { + final int n = readCount(in); + final int[] out = new int[n]; + for (int i = 0; i < n; i++) out[i] = in.getInt(); + return out; + } + + public static void writeSortedStrs(DataOutputStream out, Set values) throws IOException { + final Set sorted = values == null ? Set.of() : new TreeSet<>(values); + out.writeInt(sorted.size()); + for (String s : sorted) writeStr(out, s); + } + + public static Set readStrSet(ByteBuffer in) { + final String[] out = new String[readCount(in)]; + for (int i = 0; i < out.length; i++) out[i] = readStr(in); + return Set.copyOf(Arrays.asList(out)); + } + + public static String libraryId(SharedLibrary lib) { + final String path = lib.getPath(); + return Version.getVersion() + '|' + lib.getName() + '|' + (path == null ? -1 : new File(path).length()); + } +} diff --git a/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/shader/ShaderDiskCache.java b/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/shader/ShaderDiskCache.java new file mode 100644 index 000000000..8ea43fffb --- /dev/null +++ b/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/shader/ShaderDiskCache.java @@ -0,0 +1,307 @@ +package com.gtnewhorizons.angelica.glsm.shader; + +import com.gtnewhorizons.angelica.glsm.threading.AngelicaWorkers; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; +import java.util.zip.CRC32; + +import static com.gtnewhorizons.angelica.glsm.shader.ShaderCacheIO.readBytes; +import static com.gtnewhorizons.angelica.glsm.shader.ShaderCacheIO.readCount; +import static com.gtnewhorizons.angelica.glsm.shader.ShaderCacheIO.readStr; +import static com.gtnewhorizons.angelica.glsm.shader.ShaderCacheIO.writeBytes; +import static com.gtnewhorizons.angelica.glsm.shader.ShaderCacheIO.writeStr; + +public final class ShaderDiskCache { + private static final Logger LOGGER = LogManager.getLogger("ShaderDiskCache"); + private static final int MAGIC = 0x41534331; + private static final int HEADER_BYTES = 12; + static final long MAX_BYTES = 256L << 20; + private static volatile Path root; + private static final AtomicBoolean writeWarned = new AtomicBoolean(); + private static final AtomicInteger TMP_SEQ = new AtomicInteger(); + + private ShaderDiskCache() {} + + public static void configure(Path dir, String salt) { + try { + Files.createDirectories(dir); + final Path saltFile = dir.resolve("salt"); + final String existing = Files.isRegularFile(saltFile) ? new String(Files.readAllBytes(saltFile), StandardCharsets.UTF_8) : null; + if (!salt.equals(existing)) { + wipe(dir); + Files.write(saltFile, salt.getBytes(StandardCharsets.UTF_8)); + } + root = dir; + AngelicaWorkers.run(() -> enforceCap(dir, MAX_BYTES)); + LOGGER.info("Shader disk cache at {}", dir); + } catch (IOException | RuntimeException e) { + root = null; + LOGGER.warn("Shader disk cache disabled", e); + } + } + + public static boolean isEnabled() { + return root != null; + } + + public static Key key(String layer) { + return new Key(layer); + } + + public static byte[] get(Key key) { + final Path r = root; + if (r == null) return null; + final Path file = key.path(r); + final byte[] payload; + try (FileChannel ch = FileChannel.open(file, StandardOpenOption.READ)) { + payload = read(ch); + } catch (IOException e) { + return null; + } + if (payload == null) { + try { + Files.deleteIfExists(file); + } catch (IOException ignored) {} + } + return payload; + } + + private static byte[] read(FileChannel ch) throws IOException { + final long size = ch.size(); + if (size < HEADER_BYTES || size - HEADER_BYTES > Integer.MAX_VALUE) return null; + final ByteBuffer header = ByteBuffer.allocate(HEADER_BYTES); + readFully(ch, header); + header.flip(); + if (header.getInt() != MAGIC) return null; + final int length = header.getInt(); + final int crc = header.getInt(); + if (length != size - HEADER_BYTES) return null; + final byte[] payload = new byte[length]; + readFully(ch, ByteBuffer.wrap(payload)); + final CRC32 c = new CRC32(); + c.update(payload, 0, length); + return (int) c.getValue() == crc ? payload : null; + } + + private static void readFully(FileChannel ch, ByteBuffer buf) throws IOException { + while (buf.hasRemaining()) { + if (ch.read(buf) < 0) throw new EOFException(); + } + } + + public static void put(Key key, byte[] payload) { + final Path r = root; + if (r == null) return; + final Path target = key.path(r); + final Path tmp = target.resolveSibling(key.hex() + '.' + TMP_SEQ.getAndIncrement() + ".tmp"); + try { + try { + write(tmp, payload); + } catch (IOException e) { + Files.createDirectories(target.getParent()); + write(tmp, payload); + } + Files.move(tmp, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + try { + Files.deleteIfExists(tmp); + } catch (IOException ignored) {} + if (writeWarned.compareAndSet(false, true)) LOGGER.warn("Shader disk cache write failed for {}", target, e); + } + } + + private static void write(Path tmp, byte[] payload) throws IOException { + final CRC32 c = new CRC32(); + c.update(payload, 0, payload.length); + final ByteBuffer header = ByteBuffer.allocate(HEADER_BYTES); + header.putInt(MAGIC).putInt(payload.length).putInt((int) c.getValue()).flip(); + final ByteBuffer[] parts = { header, ByteBuffer.wrap(payload) }; + try (FileChannel ch = FileChannel.open(tmp, StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) { + while (header.hasRemaining() || parts[1].hasRemaining()) ch.write(parts); + } + } + + public static String getString(Key key) { + final byte[] payload = get(key); + return payload == null ? null : new String(payload, StandardCharsets.UTF_8); + } + + public static void putString(Key key, String value) { + put(key, value.getBytes(StandardCharsets.UTF_8)); + } + + public static Map getStrings(Key key) { + final byte[] payload = get(key); + if (payload == null) return null; + try { + final ByteBuffer in = ByteBuffer.wrap(payload); + final int count = readCount(in); + final Map out = new LinkedHashMap<>(); + for (int i = 0; i < count; i++) { + out.put(readStr(in), readStr(in)); + } + return out; + } catch (RuntimeException e) { + return null; + } + } + + public static void putStrings(Key key, Map values) { + int size = 0; + for (String v : values.values()) size += (v == null ? 0 : v.length()) + 64; + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(size); + try (DataOutputStream out = new DataOutputStream(bytes)) { + out.writeInt(values.size()); + for (Map.Entry e : values.entrySet()) { + writeStr(out, e.getKey()); + writeStr(out, e.getValue()); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + put(key, bytes.toByteArray()); + } + + public static Blob getBlob(Key key) { + final byte[] payload = get(key); + if (payload == null) return null; + try { + final ByteBuffer in = ByteBuffer.wrap(payload); + final String tag = readStr(in); + final byte[] data = readBytes(in); + return new Blob(tag, data); + } catch (RuntimeException e) { + return null; + } + } + + public static void putBlob(Key key, String tag, byte[] data) { + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(data.length + 64); + try (DataOutputStream out = new DataOutputStream(bytes)) { + writeStr(out, tag); + writeBytes(out, data); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + put(key, bytes.toByteArray()); + } + + public record Blob(String tag, byte[] data) {} + + static void enforceCap(Path dir, long maxBytes) { + try { + final long[] total = {0}; + Files.walkFileTree(dir, new SimpleFileVisitor<>() { + @Override public FileVisitResult visitFile(Path p, BasicFileAttributes a) { total[0] += a.size(); return FileVisitResult.CONTINUE; } + @Override public FileVisitResult visitFileFailed(Path p, IOException e) { return FileVisitResult.CONTINUE; } + }); + if (total[0] > maxBytes) wipe(dir); + } catch (IOException | RuntimeException e) { + LOGGER.warn("Shader disk cache cap check failed", e); + } + } + + private static void wipe(Path dir) throws IOException { + final List paths = new ArrayList<>(); + try (Stream walk = Files.walk(dir)) { + walk.forEach(paths::add); + } + for (Path p : paths) { + if (Files.isRegularFile(p) && !p.getFileName().toString().equals("salt")) Files.deleteIfExists(p); + } + } + + public static final class Key { + private final String layer; + private final MessageDigest md; + private String hex; + + Key(String layer) { + this.layer = layer; + try { + md = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + str(layer); + } + + public Key str(String s) { + if (s == null) return i(-1); + final byte[] utf8 = s.getBytes(StandardCharsets.UTF_8); + i(utf8.length); + md.update(utf8); + return this; + } + + public Key i(int v) { + md.update((byte) (v >>> 24)); + md.update((byte) (v >>> 16)); + md.update((byte) (v >>> 8)); + md.update((byte) v); + return this; + } + + public Key b(boolean v) { + md.update((byte) (v ? 1 : 0)); + return this; + } + + public Key bytes(byte[] v) { + i(v.length); + md.update(v); + return this; + } + + public Key sortedStrs(Collection values) { + final String[] sorted = values.toArray(new String[0]); + Arrays.sort(sorted); + i(sorted.length); + for (String s : sorted) str(s); + return this; + } + + public String hex() { + if (hex == null) { + final byte[] digest = md.digest(); + final StringBuilder sb = new StringBuilder(digest.length * 2); + for (byte d : digest) { + sb.append(Character.forDigit((d >>> 4) & 0xF, 16)).append(Character.forDigit(d & 0xF, 16)); + } + hex = sb.toString(); + } + return hex; + } + + Path path(Path root) { + return root.resolve(layer).resolve(hex() + ".bin"); + } + } +} diff --git a/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/shader/SpirvCompiler.java b/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/shader/SpirvCompiler.java index b0a95b0e1..87d26e158 100644 --- a/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/shader/SpirvCompiler.java +++ b/glsm/src/main/java/com/gtnewhorizons/angelica/glsm/shader/SpirvCompiler.java @@ -35,6 +35,11 @@ public final class SpirvCompiler { private static final int CACHE_MAX = 256; private static final Object2ObjectLinkedOpenHashMap CACHE = new Object2ObjectLinkedOpenHashMap<>(); + @Lwjgl3Aware + private static final class Toolchain { + static final String ID = ShaderCacheIO.libraryId(Shaderc.getLibrary()); + } + private record CacheKey(String source, int shaderKind, Options opts) {} private SpirvCompiler() {} @@ -53,6 +58,10 @@ public final class SpirvCompiler { public record Result(@Nullable ByteBuffer spirv, @Nullable String error, @Nullable Path dumpPath) {} public static Result compile(String source, int shaderKind, String debugName, Options opts) { + return compile(source, shaderKind, debugName, opts, true); + } + + public static Result compile(String source, int shaderKind, String debugName, Options opts, boolean diskCache) { final String cleanSrc = source.indexOf('\0') >= 0 ? source.replace("\0", "") : source; final CacheKey key = new CacheKey(cleanSrc, shaderKind, opts); @@ -60,12 +69,37 @@ public final class SpirvCompiler { synchronized (CACHE) { cached = CACHE.getAndMoveToFirst(key); } - if (cached != null) { - final ByteBuffer copy = MemoryUtil.memAlloc(cached.length); - copy.put(cached).flip(); - return new Result(copy, null, null); + if (cached != null) return new Result(ShaderCacheIO.toNative(cached), null, null); + + ShaderDiskCache.Key diskKey = null; + if (diskCache && ShaderDiskCache.isEnabled()) { + diskKey = diskKey(cleanSrc, shaderKind, opts); + final byte[] disk = ShaderDiskCache.get(diskKey); + if (disk != null) { + ShaderCacheIO.lruPut(CACHE, key, disk, CACHE_MAX); + return new Result(ShaderCacheIO.toNative(disk), null, null); + } } + final Result result = compileUncached(cleanSrc, shaderKind, debugName, opts); + final ByteBuffer spirv = result.spirv(); + if (spirv != null) { + final byte[] heap = ShaderCacheIO.toHeap(spirv); + ShaderCacheIO.lruPut(CACHE, key, heap, CACHE_MAX); + if (diskKey != null) ShaderDiskCache.put(diskKey, heap); + } + return result; + } + + public static String toolchainId() { + return Toolchain.ID; + } + + static ShaderDiskCache.Key diskKey(String cleanSrc, int shaderKind, Options opts) { + return ShaderDiskCache.key("spirv").str(toolchainId()).str(cleanSrc).i(shaderKind).str(opts.toString()); + } + + private static Result compileUncached(String cleanSrc, int shaderKind, String debugName, Options opts) { final long compiler = COMPILER.get(); final long optionsHandle = Shaderc.shaderc_compile_options_initialize(); if (optionsHandle == 0L) { @@ -98,12 +132,6 @@ public final class SpirvCompiler { final ByteBuffer copy = MemoryUtil.memAlloc(spirv.remaining()); copy.put(spirv); copy.flip(); - final byte[] heap = new byte[copy.remaining()]; - copy.duplicate().get(heap); - synchronized (CACHE) { - CACHE.putAndMoveToFirst(key, heap); - while (CACHE.size() > CACHE_MAX) CACHE.removeLast(); - } return new Result(copy, null, null); } finally { if (result != 0L) Shaderc.shaderc_result_release(result); diff --git a/glsm/src/test/java/com/gtnewhorizons/angelica/glsm/CompatShaderTransformerTest.java b/glsm/src/test/java/com/gtnewhorizons/angelica/glsm/CompatShaderTransformerTest.java index 89b65320a..0ff1deaba 100644 --- a/glsm/src/test/java/com/gtnewhorizons/angelica/glsm/CompatShaderTransformerTest.java +++ b/glsm/src/test/java/com/gtnewhorizons/angelica/glsm/CompatShaderTransformerTest.java @@ -1,9 +1,14 @@ package com.gtnewhorizons.angelica.glsm; +import com.gtnewhorizons.angelica.glsm.shader.ShaderDiskCache; +import com.gtnewhorizons.angelica.glsm.testutil.Reflect; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import java.nio.file.Path; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -872,4 +877,28 @@ class CompatShaderTransformerTest { assertNoLoc(c, 1, "vec4 a_y"); // must not collide with the hex-reserved slot assertLoc(c, 0, "vec4 a_y"); } + + @Test + void diskCacheHitReplacesTransform(@TempDir Path dir) { + final String src = """ + #version 120 + void main() { + gl_Position = gl_ModelViewProjectionMatrix * gl_Vertex; + } + """; + try { + ShaderDiskCache.configure(dir, "t"); + CompatShaderTransformer.clearCache(); + final String first = CompatShaderTransformer.transform(src, false); + assertTrue(first.contains("angelica_ProjectionMatrix")); + CompatShaderTransformer.clearCache(); + assertEquals(first, CompatShaderTransformer.transform(src, false)); + CompatShaderTransformer.clearCache(); + ShaderDiskCache.putString(CompatShaderTransformer.diskKey(src, false), "POISON"); + assertEquals("POISON", CompatShaderTransformer.transform(src, false)); + } finally { + CompatShaderTransformer.clearCache(); + Reflect.setStatic(ShaderDiskCache.class, "root", null); + } + } } diff --git a/glsm/src/test/java/com/gtnewhorizons/angelica/glsm/shader/ShaderDiskCacheTest.java b/glsm/src/test/java/com/gtnewhorizons/angelica/glsm/shader/ShaderDiskCacheTest.java new file mode 100644 index 000000000..1dc8a13b8 --- /dev/null +++ b/glsm/src/test/java/com/gtnewhorizons/angelica/glsm/shader/ShaderDiskCacheTest.java @@ -0,0 +1,125 @@ +package com.gtnewhorizons.angelica.glsm.shader; + +import com.gtnewhorizons.angelica.glsm.testutil.Reflect; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ShaderDiskCacheTest { + + @TempDir + Path dir; + + @AfterEach + void reset() { + Reflect.setStatic(ShaderDiskCache.class, "root", null); + } + + private static long binCount(Path dir) throws IOException { + try (Stream walk = Files.walk(dir)) { + return walk.filter(p -> p.getFileName().toString().endsWith(".bin")).count(); + } + } + + @Test + void roundTripAndDisabled() throws IOException { + ShaderDiskCache.put(ShaderDiskCache.key("l").str("x"), new byte[] {1}); + assertNull(ShaderDiskCache.get(ShaderDiskCache.key("l").str("x"))); + assertEquals(0, binCount(dir)); + + ShaderDiskCache.configure(dir, "a"); + ShaderDiskCache.put(ShaderDiskCache.key("l").str("x"), new byte[] {1, 2, 3}); + assertArrayEquals(new byte[] {1, 2, 3}, ShaderDiskCache.get(ShaderDiskCache.key("l").str("x"))); + assertNull(ShaderDiskCache.get(ShaderDiskCache.key("l").str("y"))); + + ShaderDiskCache.put(ShaderDiskCache.key("t").str("empty"), new byte[0]); + assertArrayEquals(new byte[0], ShaderDiskCache.get(ShaderDiskCache.key("t").str("empty"))); + } + + @Test + void saltChangeWipes() { + ShaderDiskCache.configure(dir, "a"); + ShaderDiskCache.put(ShaderDiskCache.key("l").str("x"), new byte[] {7}); + ShaderDiskCache.configure(dir, "a"); + assertArrayEquals(new byte[] {7}, ShaderDiskCache.get(ShaderDiskCache.key("l").str("x"))); + ShaderDiskCache.configure(dir, "b"); + assertNull(ShaderDiskCache.get(ShaderDiskCache.key("l").str("x"))); + } + + @Test + void corruptAndTruncatedAreMisses() throws IOException { + ShaderDiskCache.configure(dir, "a"); + final ShaderDiskCache.Key k = ShaderDiskCache.key("l").str("x"); + ShaderDiskCache.put(k, new byte[] {1, 2, 3, 4}); + final Path file = k.path(dir); + final byte[] raw = Files.readAllBytes(file); + raw[raw.length - 1] ^= 1; + Files.write(file, raw); + assertNull(ShaderDiskCache.get(ShaderDiskCache.key("l").str("x"))); + assertFalse(Files.exists(file)); + + ShaderDiskCache.put(ShaderDiskCache.key("l").str("x"), new byte[] {1, 2, 3, 4}); + final byte[] full = Files.readAllBytes(file); + final byte[] cut = new byte[full.length - 2]; + System.arraycopy(full, 0, cut, 0, cut.length); + Files.write(file, cut); + assertNull(ShaderDiskCache.get(ShaderDiskCache.key("l").str("x"))); + + final ShaderDiskCache.Key shortKey = ShaderDiskCache.key("t").str("short"); + ShaderDiskCache.put(shortKey, new byte[] {1}); + final Path shortFile = shortKey.path(dir); + Files.write(shortFile, new byte[] {0x41, 0x53}); + assertNull(ShaderDiskCache.get(ShaderDiskCache.key("t").str("short"))); + assertFalse(Files.exists(shortFile)); + } + + @Test + void keysAreFramed() { + assertNotEquals(ShaderDiskCache.key("l").str("ab").str("c").hex(), ShaderDiskCache.key("l").str("a").str("bc").hex()); + assertNotEquals(ShaderDiskCache.key("l").str(null).hex(), ShaderDiskCache.key("l").str("").hex()); + } + + @Test + void stringsAndBlobs() { + ShaderDiskCache.configure(dir, "a"); + final Map values = new LinkedHashMap<>(); + values.put("VERTEX", null); + values.put("FRAGMENT", "x".repeat(100000)); + ShaderDiskCache.putStrings(ShaderDiskCache.key("s").str("1"), values); + assertEquals(values, ShaderDiskCache.getStrings(ShaderDiskCache.key("s").str("1"))); + + ShaderDiskCache.putBlob(ShaderDiskCache.key("b").str("1"), "main0", new byte[] {9, 8, 7}); + final ShaderDiskCache.Blob blob = ShaderDiskCache.getBlob(ShaderDiskCache.key("b").str("1")); + assertNotNull(blob); + assertEquals("main0", blob.tag()); + assertArrayEquals(new byte[] {9, 8, 7}, blob.data()); + } + + @Test + void overCapWipesEntriesAndKeepsSalt() throws IOException { + ShaderDiskCache.configure(dir, "a"); + for (int i = 0; i < 4; i++) ShaderDiskCache.put(ShaderDiskCache.key("t").i(i), new byte[1000]); + ShaderDiskCache.enforceCap(dir, 1 << 20); + assertEquals(4, binCount(dir)); + ShaderDiskCache.enforceCap(dir, 3000); + assertEquals(0, binCount(dir)); + assertTrue(Files.isRegularFile(dir.resolve("salt"))); + ShaderDiskCache.put(ShaderDiskCache.key("t").i(9), new byte[] {5}); + assertArrayEquals(new byte[] {5}, ShaderDiskCache.get(ShaderDiskCache.key("t").i(9))); + } +} diff --git a/glsm/src/test/java/com/gtnewhorizons/angelica/glsm/shader/SpirvCompilerDiskCacheTest.java b/glsm/src/test/java/com/gtnewhorizons/angelica/glsm/shader/SpirvCompilerDiskCacheTest.java new file mode 100644 index 000000000..b860f6d79 --- /dev/null +++ b/glsm/src/test/java/com/gtnewhorizons/angelica/glsm/shader/SpirvCompilerDiskCacheTest.java @@ -0,0 +1,61 @@ +package com.gtnewhorizons.angelica.glsm.shader; + +import com.gtnewhorizons.angelica.glsm.testutil.Reflect; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.lwjgl.system.MemoryUtil; +import org.lwjgl.util.shaderc.Shaderc; + +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +@Tag("lwjgl3") +class SpirvCompilerDiskCacheTest { + + private static final String SRC = "#version 460 core\nlayout(location = 0) in vec4 p;\nvoid main() { gl_Position = p; }\n"; + + @TempDir + Path dir; + + @AfterEach + void reset() { + Reflect.setStatic(ShaderDiskCache.class, "root", null); + SpirvCompiler.clearCache(); + } + + @Test + void diskEntryServedAfterMemoryClear() { + final SpirvCompiler.Options opts = SpirvCompiler.Options.vulkanForced460Core(); + ShaderDiskCache.configure(dir, "t"); + final SpirvCompiler.Result first = SpirvCompiler.compile(SRC, Shaderc.shaderc_vertex_shader, "disk-cache-test", opts); + assertNotNull(first.spirv(), first.error()); + MemoryUtil.memFree(first.spirv()); + + SpirvCompiler.clearCache(); + ShaderDiskCache.put(SpirvCompiler.diskKey(SRC, Shaderc.shaderc_vertex_shader, opts), new byte[] {1, 2, 3, 4}); + final SpirvCompiler.Result second = SpirvCompiler.compile(SRC, Shaderc.shaderc_vertex_shader, "disk-cache-test", opts); + final ByteBuffer spirv = second.spirv(); + assertNotNull(spirv); + try { + assertArrayEquals(new byte[] {1, 2, 3, 4}, ShaderCacheIO.toHeap(spirv)); + } finally { + MemoryUtil.memFree(spirv); + } + } + + @Test + void diskOptOutSkipsSpirvLayer() { + ShaderDiskCache.configure(dir, "t"); + final SpirvCompiler.Result r = SpirvCompiler.compile(SRC, Shaderc.shaderc_vertex_shader, "disk-opt-out", SpirvCompiler.Options.vulkanForced460Core(), false); + assertNotNull(r.spirv(), r.error()); + MemoryUtil.memFree(r.spirv()); + assertFalse(Files.exists(dir.resolve("spirv"))); + } +} diff --git a/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/ClipZRemap.java b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/ClipZRemap.java index 66cec9191..1d4f962ed 100644 --- a/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/ClipZRemap.java +++ b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/ClipZRemap.java @@ -1,34 +1,18 @@ package com.gtnewhorizons.angelica.sdlgpu.shader; -import com.gtnewhorizons.angelica.glsm.GlslTransformUtils; import com.gtnewhorizons.angelica.glsm.shader.GlslVulkanPreprocess.Edit; import org.antlr.v4.runtime.tree.ParseTreeWalker; import org.taumc.glsl.grammar.GLSLParser; import org.taumc.glsl.grammar.GLSLParserBaseListener; -import java.util.ArrayList; import java.util.List; -import static com.gtnewhorizons.angelica.glsm.shader.GlslVulkanPreprocess.applyEdits; - public final class ClipZRemap { private static final String INJECTION_STD = "gl_Position.z = gl_Position.z * 0.5 + gl_Position.w * 0.5;\n"; private ClipZRemap() {} - public static String injectGLToVulkanClipZ(String source) { - final GLSLParser.Translation_unitContext root; - try { - root = GlslTransformUtils.parseFullQuiet(source); - } catch (Exception e) { - return source; - } - final ArrayList edits = new ArrayList<>(1); - collectEdits(root, edits); - return edits.isEmpty() ? source : applyEdits(source, edits); - } - public static void collectEdits(GLSLParser.Translation_unitContext root, List edits) { final int[] mainCloseBraceIdx = { -1 }; ParseTreeWalker.DEFAULT.walk(new GLSLParserBaseListener() { diff --git a/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/PerFrameBlockInjector.java b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/PerFrameBlockInjector.java index 49dd71e3c..bd89ec97b 100644 --- a/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/PerFrameBlockInjector.java +++ b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/PerFrameBlockInjector.java @@ -1,6 +1,5 @@ package com.gtnewhorizons.angelica.sdlgpu.shader; -import com.gtnewhorizons.angelica.glsm.GlslTransformUtils; import com.gtnewhorizons.angelica.glsm.hooks.PerFrameUniformBlock; import com.gtnewhorizons.angelica.glsm.hooks.PerFrameUniformBlock.Member; import com.gtnewhorizons.angelica.glsm.shader.GlslVulkanPreprocess.Edit; @@ -22,7 +21,6 @@ import java.util.Map; import java.util.List; import java.util.Set; -import static com.gtnewhorizons.angelica.glsm.shader.GlslVulkanPreprocess.applyEdits; public final class PerFrameBlockInjector { @@ -30,21 +28,6 @@ public final class PerFrameBlockInjector { private PerFrameBlockInjector() {} - public static String inject(String source, PerFrameUniformBlock perFrame, PerFrameUniformBlock perPass) { - if (normalize(perFrame) == null && normalize(perPass) == null) return source; - - final GLSLParser.Translation_unitContext root; - try { - root = GlslTransformUtils.parseFullQuiet(source); - } catch (Exception e) { - return source; - } - - final List edits = new ArrayList<>(); - collectEdits(root, perFrame, perPass, edits); - return edits.isEmpty() ? source : applyEdits(source, edits); - } - public static void collectEdits(GLSLParser.Translation_unitContext root, PerFrameUniformBlock perFrame, PerFrameUniformBlock perPass, List out) { final PerFrameUniformBlock[] blocks = new PerFrameUniformBlock[ShaderManager.BLOCK_COUNT]; blocks[ShaderManager.BLOCK_PER_FRAME] = normalize(perFrame); diff --git a/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/SamplerStripper.java b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/SamplerStripper.java index 073f930f9..da9264212 100644 --- a/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/SamplerStripper.java +++ b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/SamplerStripper.java @@ -17,7 +17,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import static com.gtnewhorizons.angelica.glsm.shader.GlslVulkanPreprocess.applyEdits; public final class SamplerStripper { @@ -25,20 +24,6 @@ public final class SamplerStripper { private SamplerStripper() {} - public static String stripUnused(String source) { - if (!source.contains("sampler")) return source; - - final GLSLParser.Translation_unitContext root; - try { - root = GlslTransformUtils.parseFullQuiet(source); - } catch (Exception e) { - return source; - } - final List edits = new ArrayList<>(); - collectEdits(root, source, edits); - return edits.isEmpty() ? source : applyEdits(source, edits); - } - public static void collectEdits(GLSLParser.Translation_unitContext root, String source, List edits) { record SamplerDecl(String name, int start, int stop) {} final List samplers = new ArrayList<>(); diff --git a/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderManager.java b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderManager.java index a35ac603a..99d4d257a 100644 --- a/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderManager.java +++ b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderManager.java @@ -4,12 +4,16 @@ import com.gtnewhorizons.angelica.config.SystemProperties; import com.gtnewhorizons.angelica.glsm.GlslTransformUtils; import com.gtnewhorizons.angelica.glsm.hooks.GLSMHooks; import com.gtnewhorizons.angelica.glsm.hooks.PerFrameUniformBlock; +import com.gtnewhorizons.angelica.glsm.hooks.PerFrameUniformBlock.Member; import com.gtnewhorizons.angelica.glsm.shader.GlslVulkanPreprocess; import com.gtnewhorizons.angelica.glsm.shader.GlslVulkanPreprocess.Edit; +import com.gtnewhorizons.angelica.glsm.shader.ShaderCacheIO; +import com.gtnewhorizons.angelica.glsm.shader.ShaderDiskCache; import com.gtnewhorizons.angelica.glsm.shader.SpirvCompiler; import com.gtnewhorizons.angelica.glsm.threading.AngelicaWorkers; import com.gtnewhorizons.angelica.sdlgpu.device.Device; import com.gtnewhorizons.angelica.sdlgpu.frame.ContextState; +import com.gtnewhorizons.angelica.sdlgpu.shader.cross.CrossCompileCache; import org.taumc.glsl.grammar.GLSLParser; import com.gtnewhorizons.angelica.sdlgpu.shader.cross.CrossCompileUtil; import com.gtnewhorizons.angelica.sdlgpu.shader.dxbc.DxbcCrossCompile; @@ -105,32 +109,74 @@ public final class ShaderManager { return id; } - private record PrewarmKey(String source, int glShaderType) {} - private record PrewarmEntry(byte[] remappedSpirv, StageReflection reflection, GraphicsBindingMap graphicsBindingMap, Set boolUniforms, String transformedSource) {} + private record SourceKey(String source, int glShaderType, PerFrameUniformBlock perFrame, PerFrameUniformBlock perPass) {} private record PrewarmHit(ByteBuffer spirv, StageReflection reflection, GraphicsBindingMap graphicsBindingMap, Set boolUniforms, String transformedSource) {} private static final int PREWARM_CACHE_MAX = 256; - private static final Object2ObjectLinkedOpenHashMap PREWARM_CACHE = new Object2ObjectLinkedOpenHashMap<>(); + private static final Object2ObjectLinkedOpenHashMap PREWARM_CACHE = new Object2ObjectLinkedOpenHashMap<>(); - private record TransformKey(String source, int glShaderType, PerFrameUniformBlock perFrame, PerFrameUniformBlock perPass) {} - private record TransformEntry(String source, Set boolUniforms) {} private static final int TRANSFORM_CACHE_MAX = 256; - private static final Object2ObjectLinkedOpenHashMap TRANSFORM_CACHE = new Object2ObjectLinkedOpenHashMap<>(); + private static final Object2ObjectLinkedOpenHashMap TRANSFORM_CACHE = new Object2ObjectLinkedOpenHashMap<>(); public static void clearPrewarmCache() { synchronized (PREWARM_CACHE) { PREWARM_CACHE.clear(); } synchronized (TRANSFORM_CACHE) { TRANSFORM_CACHE.clear(); } } + static ShaderDiskCache.Key stageKey(String source, int glShaderType) { + return ShaderDiskCache.key("sdl-stage").str(SpirvCompiler.toolchainId()).str(CrossCompileUtil.spvcId()).str(source).i(glShaderType); + } + + static ShaderDiskCache.Key prewarmKey(String source, int glShaderType, PerFrameUniformBlock perFrame, PerFrameUniformBlock perPass) { + return appendBlocks(ShaderDiskCache.key("sdl-prewarm").str(SpirvCompiler.toolchainId()).str(CrossCompileUtil.spvcId()).str(source).i(glShaderType), perFrame, perPass); + } + + static ShaderDiskCache.Key transformDiskKey(String source, int glShaderType, PerFrameUniformBlock perFrame, PerFrameUniformBlock perPass) { + return appendBlocks(ShaderDiskCache.key("sdl-transform").str(source).i(glShaderType), perFrame, perPass); + } + + private static ShaderDiskCache.Key appendBlocks(ShaderDiskCache.Key key, PerFrameUniformBlock perFrame, PerFrameUniformBlock perPass) { + appendBlock(key, perFrame); + appendBlock(key, perPass); + return key; + } + + private static void appendBlock(ShaderDiskCache.Key key, PerFrameUniformBlock block) { + if (block == null) { + key.i(-1); + return; + } + key.i(block.members().size()); + for (Member m : block.members()) key.str(m.name()).str(m.type().name()); + } + + private static boolean isGraphics(int glShaderType) { + return glShaderType == GL20.GL_VERTEX_SHADER || glShaderType == GL20.GL_FRAGMENT_SHADER; + } + + private static ShaderStageSerializer.Stage loadStage(ShaderDiskCache.Key key) { + final byte[] payload = ShaderDiskCache.get(key); + return payload == null ? null : ShaderStageSerializer.decode(payload); + } + + private static PrewarmTransformResult loadTransform(ShaderDiskCache.Key key) { + final byte[] payload = ShaderDiskCache.get(key); + return payload == null ? null : ShaderStageSerializer.decodeTransform(payload); + } + + private static void warnSeparateSamplerOnHit(StageReflection reflection, int glShaderType) { + if (!reflection.extraUniformNames().isEmpty()) { + warnSeparateSampler(glShaderType == GL20.GL_VERTEX_SHADER ? "vertex" : "fragment"); + } + } + private static PrewarmHit lookupPrewarm(String src, int glShaderType) { - final PrewarmEntry e; + final ShaderStageSerializer.Stage e; synchronized (PREWARM_CACHE) { - e = PREWARM_CACHE.getAndMoveToFirst(new PrewarmKey(src, glShaderType)); + e = PREWARM_CACHE.getAndMoveToFirst(new SourceKey(src, glShaderType, GLSMHooks.perFrameUniformBlock, GLSMHooks.perPassUniformBlock)); } if (e == null) return null; // per-hit copy: linkProgram patches the returned SPIR-V in place (applyVaryingMatch / applyAttribLocationsAndInputMask) - final ByteBuffer copy = memAlloc(e.remappedSpirv().length); - copy.put(e.remappedSpirv()).flip(); - return new PrewarmHit(copy, e.reflection(), e.graphicsBindingMap(), e.boolUniforms(), e.transformedSource()); + return new PrewarmHit(ShaderCacheIO.toNative(e.spirv()), e.reflection(), e.bindingMap(), e.boolUniforms(), e.source()); } public ShaderManager(Device device) { @@ -165,26 +211,22 @@ public final class ShaderManager { return; } - final TransformKey transformKey = new TransformKey(raw, obj.type, GLSMHooks.perFrameUniformBlock, GLSMHooks.perPassUniformBlock); - TransformEntry transformed; + final PerFrameUniformBlock perFrame = GLSMHooks.perFrameUniformBlock; + final PerFrameUniformBlock perPass = GLSMHooks.perPassUniformBlock; + final SourceKey transformKey = new SourceKey(raw, obj.type, perFrame, perPass); + PrewarmTransformResult transformed; synchronized (TRANSFORM_CACHE) { transformed = TRANSFORM_CACHE.getAndMoveToFirst(transformKey); } if (transformed == null) { - final GlslVulkanPreprocess.Result pre = GlslVulkanPreprocess.run(raw, obj.type, "shader" + shader, true); - String src = pre != null ? pre.rewrittenSource() : raw; - if (obj.isVertex()) { - src = ClipZRemap.injectGLToVulkanClipZ(src); - } - src = SamplerStripper.stripUnused(src); - if (obj.type == GL20.GL_VERTEX_SHADER || obj.type == GL20.GL_FRAGMENT_SHADER) { - src = PerFrameBlockInjector.inject(src, GLSMHooks.perFrameUniformBlock, GLSMHooks.perPassUniformBlock); - } - transformed = new TransformEntry(src, pre != null ? pre.boolUniforms() : Set.of()); - synchronized (TRANSFORM_CACHE) { - TRANSFORM_CACHE.putAndMoveToFirst(transformKey, transformed); - while (TRANSFORM_CACHE.size() > TRANSFORM_CACHE_MAX) TRANSFORM_CACHE.removeLast(); + final ShaderDiskCache.Key diskKey = ShaderDiskCache.isEnabled() ? transformDiskKey(raw, obj.type, perFrame, perPass) : null; + if (diskKey != null) transformed = loadTransform(diskKey); + if (transformed == null) { + transformed = transformOrNull(raw, obj.type, "shader" + shader, perFrame, perPass); + if (transformed == null) transformed = new PrewarmTransformResult(raw, Set.of()); + else if (diskKey != null) ShaderDiskCache.put(diskKey, ShaderStageSerializer.encodeTransform(transformed.source(), transformed.boolUniforms())); } + ShaderCacheIO.lruPut(TRANSFORM_CACHE, transformKey, transformed, TRANSFORM_CACHE_MAX); } obj.boolUniforms = transformed.boolUniforms(); obj.source = transformed.source(); @@ -192,13 +234,29 @@ public final class ShaderManager { final int shaderKind = shaderKindFor(obj); final int glType = obj.type; final String finalSrc = obj.source; - obj.spirvFuture = AngelicaWorkers.submit(() -> { - final SpirvCompiler.Result r = SpirvCompiler.compile(finalSrc, shaderKind, "shader" + shader, SpirvCompiler.Options.vulkanForced460Core()); - if (r.spirv() != null && (glType == GL20.GL_VERTEX_SHADER || glType == GL20.GL_FRAGMENT_SHADER)) { - return new AsyncCompile(r, remapSpirvForSDLGPU(r.spirv(), glType), reflectStage(r.spirv(), glType == GL20.GL_VERTEX_SHADER)); + obj.spirvFuture = AngelicaWorkers.submit(() -> compileStageAsync(finalSrc, shaderKind, glType, shader)); + } + + private static AsyncCompile compileStageAsync(String src, int shaderKind, int glType, int shader) { + final boolean graphics = isGraphics(glType); + final ShaderDiskCache.Key diskKey = graphics && ShaderDiskCache.isEnabled() ? stageKey(src, glType) : null; + if (diskKey != null) { + final ShaderStageSerializer.Stage d = loadStage(diskKey); + if (d != null) { + warnSeparateSamplerOnHit(d.reflection(), glType); + return new AsyncCompile(new SpirvCompiler.Result(ShaderCacheIO.toNative(d.spirv()), null, null), d.bindingMap(), d.reflection()); } - return new AsyncCompile(r, null, null); - }); + } + final SpirvCompiler.Result r = SpirvCompiler.compile(src, shaderKind, "shader" + shader, SpirvCompiler.Options.vulkanForced460Core(), diskKey == null); + if (r.spirv() != null && graphics) { + final GraphicsBindingMap map = remapSpirvForSDLGPU(r.spirv(), glType); + final StageReflection reflection = reflectStage(r.spirv(), glType == GL20.GL_VERTEX_SHADER); + if (diskKey != null && reflection != StageReflection.EMPTY) { + ShaderDiskCache.put(diskKey, ShaderStageSerializer.encode(ShaderCacheIO.toHeap(r.spirv()), reflection, map, null, null)); + } + return new AsyncCompile(r, map, reflection); + } + return new AsyncCompile(r, null, null); } private static int shaderKindFor(ShaderObject obj) { @@ -214,22 +272,24 @@ public final class ShaderManager { public record PrewarmTransformResult(String source, Set boolUniforms) {} - public static String applyPrewarmTransforms(String transformedSource, int glShaderType) { - return applyPrewarmTransformsFull(transformedSource, glShaderType).source(); + public static PrewarmTransformResult applyPrewarmTransformsFull(String transformedSource, int glShaderType, PerFrameUniformBlock perFrame, PerFrameUniformBlock perPass) { + final PrewarmTransformResult r = transformOrNull(transformedSource, glShaderType, "prewarm", perFrame, perPass); + return r != null ? r : new PrewarmTransformResult(transformedSource, Set.of()); } - public static PrewarmTransformResult applyPrewarmTransformsFull(String transformedSource, int glShaderType) { + private static PrewarmTransformResult transformOrNull(String transformedSource, int glShaderType, String debugName, PerFrameUniformBlock perFrame, PerFrameUniformBlock perPass) { final GLSLParser.Translation_unitContext root; try { root = GlslTransformUtils.parseFullQuiet(transformedSource); } catch (Exception e) { - return new PrewarmTransformResult(transformedSource, Set.of()); + LOG.warn("glsl-transformation-lib parse failed for '{}': {}", debugName, e.getMessage()); + return null; } final List edits = new ArrayList<>(); - if (glShaderType == GL20.GL_VERTEX_SHADER || glShaderType == GL20.GL_FRAGMENT_SHADER) { - PerFrameBlockInjector.collectEdits(root, GLSMHooks.perFrameUniformBlock, GLSMHooks.perPassUniformBlock, edits); + if (isGraphics(glShaderType)) { + PerFrameBlockInjector.collectEdits(root, perFrame, perPass, edits); } - final GlslVulkanPreprocess.Metadata meta = GlslVulkanPreprocess.collectEdits(transformedSource, root, glShaderType, "prewarm", true, edits); + final GlslVulkanPreprocess.Metadata meta = GlslVulkanPreprocess.collectEdits(transformedSource, root, glShaderType, debugName, true, edits); if (glShaderType == GL20.GL_VERTEX_SHADER) { ClipZRemap.collectEdits(root, edits); } @@ -238,12 +298,12 @@ public final class ShaderManager { return new PrewarmTransformResult(s, meta.boolUniforms()); } - public static PrewarmTransformResult applyPrewarmTransformsFull(String finalSource, GLSLParser.Translation_unitContext bodyTree, int headerLen, int glShaderType) { + public static PrewarmTransformResult applyPrewarmTransformsFull(String finalSource, GLSLParser.Translation_unitContext bodyTree, int headerLen, int glShaderType, PerFrameUniformBlock perFrame, PerFrameUniformBlock perPass) { final String header = finalSource.substring(0, headerLen); final String body = finalSource.substring(headerLen); final List edits = new ArrayList<>(); - if (glShaderType == GL20.GL_VERTEX_SHADER || glShaderType == GL20.GL_FRAGMENT_SHADER) { - PerFrameBlockInjector.collectEdits(bodyTree, GLSMHooks.perFrameUniformBlock, GLSMHooks.perPassUniformBlock, edits); + if (isGraphics(glShaderType)) { + PerFrameBlockInjector.collectEdits(bodyTree, perFrame, perPass, edits); } final GlslVulkanPreprocess.Metadata meta = GlslVulkanPreprocess.collectEdits(null, bodyTree, glShaderType, "prewarm", true, edits); if (glShaderType == GL20.GL_VERTEX_SHADER) { @@ -271,31 +331,39 @@ public final class ShaderManager { return; } - final PrewarmKey key = new PrewarmKey(transformedSource, glShaderType); + final PerFrameUniformBlock perFrame = GLSMHooks.perFrameUniformBlock; + final PerFrameUniformBlock perPass = GLSMHooks.perPassUniformBlock; + final SourceKey key = new SourceKey(transformedSource, glShaderType, perFrame, perPass); synchronized (PREWARM_CACHE) { if (PREWARM_CACHE.containsKey(key)) return; } - final PrewarmTransformResult pre = bodyTree != null - ? applyPrewarmTransformsFull(transformedSource, bodyTree, headerLen, glShaderType) - : applyPrewarmTransformsFull(transformedSource, glShaderType); + final boolean graphics = isGraphics(glShaderType); + final ShaderDiskCache.Key diskKey = ShaderDiskCache.isEnabled() ? prewarmKey(transformedSource, glShaderType, perFrame, perPass) : null; + if (diskKey != null) { + final ShaderStageSerializer.Stage d = loadStage(diskKey); + if (d != null) { + if (graphics) warnSeparateSamplerOnHit(d.reflection(), glShaderType); + ShaderCacheIO.lruPut(PREWARM_CACHE, key, graphics ? d : new ShaderStageSerializer.Stage(d.spirv(), StageReflection.EMPTY, GraphicsBindingMap.EMPTY, d.boolUniforms(), d.source()), PREWARM_CACHE_MAX); + return; + } + } - final SpirvCompiler.Result r = SpirvCompiler.compile(pre.source(), shaderKindFor(glShaderType), "prewarm", SpirvCompiler.Options.vulkanForced460Core()); + final PrewarmTransformResult pre = bodyTree != null + ? applyPrewarmTransformsFull(transformedSource, bodyTree, headerLen, glShaderType, perFrame, perPass) + : applyPrewarmTransformsFull(transformedSource, glShaderType, perFrame, perPass); + + final SpirvCompiler.Result r = SpirvCompiler.compile(pre.source(), shaderKindFor(glShaderType), "prewarm", SpirvCompiler.Options.vulkanForced460Core(), diskKey == null); if (r.spirv() == null) return; try { - GraphicsBindingMap graphicsBindingMap = GraphicsBindingMap.EMPTY; - if (glShaderType == GL20.GL_VERTEX_SHADER || glShaderType == GL20.GL_FRAGMENT_SHADER) { - graphicsBindingMap = remapSpirvForSDLGPU(r.spirv(), glShaderType); - } - final StageReflection reflection = (glShaderType == GL20.GL_VERTEX_SHADER || glShaderType == GL20.GL_FRAGMENT_SHADER) - ? reflectStage(r.spirv(), glShaderType == GL20.GL_VERTEX_SHADER) : StageReflection.EMPTY; + final GraphicsBindingMap graphicsBindingMap = graphics ? remapSpirvForSDLGPU(r.spirv(), glShaderType) : GraphicsBindingMap.EMPTY; + final StageReflection reflection = graphics ? reflectStage(r.spirv(), glShaderType == GL20.GL_VERTEX_SHADER) : StageReflection.EMPTY; - final byte[] heap = new byte[r.spirv().remaining()]; - r.spirv().duplicate().get(heap); - synchronized (PREWARM_CACHE) { - PREWARM_CACHE.putAndMoveToFirst(key, new PrewarmEntry(heap, reflection, graphicsBindingMap, pre.boolUniforms(), pre.source())); - while (PREWARM_CACHE.size() > PREWARM_CACHE_MAX) PREWARM_CACHE.removeLast(); + final byte[] heap = ShaderCacheIO.toHeap(r.spirv()); + ShaderCacheIO.lruPut(PREWARM_CACHE, key, new ShaderStageSerializer.Stage(heap, reflection, graphicsBindingMap, pre.boolUniforms(), pre.source()), PREWARM_CACHE_MAX); + if (diskKey != null && (!graphics || reflection != StageReflection.EMPTY)) { + ShaderDiskCache.put(diskKey, ShaderStageSerializer.encode(heap, reflection, graphicsBindingMap, pre.boolUniforms(), pre.source())); } } finally { memFree(r.spirv()); @@ -327,7 +395,7 @@ public final class ShaderManager { } if (obj.spirv != null && obj.spirv != r.spirv()) memFree(obj.spirv); obj.spirv = r.spirv(); - if (!alreadyRemapped && (obj.type == GL20.GL_VERTEX_SHADER || obj.type == GL20.GL_FRAGMENT_SHADER)) { + if (!alreadyRemapped && isGraphics(obj.type)) { obj.graphicsBindingMap = remapSpirvForSDLGPU(obj.spirv, obj.type); obj.reflection = reflectStage(obj.spirv, obj.isVertex()); } @@ -1033,11 +1101,11 @@ public final class ShaderManager { return new CrossCompiled(SDL_GPU_SHADERFORMAT_SPIRV, spirv, "main", false); } if (device.supportsMsl()) { - final MslCrossCompile.Output out = MslCrossCompile.compile(spirv, glShaderType); + final CrossCompileCache.Output out = MslCrossCompile.compile(spirv, glShaderType); return new CrossCompiled(SDL_GPU_SHADERFORMAT_MSL, out.code(), out.entrypoint(), true); } if (device.supportsDxbc()) { - final DxbcCrossCompile.Output out = DxbcCrossCompile.compile(spirv, glShaderType); + final CrossCompileCache.Output out = DxbcCrossCompile.compile(spirv, glShaderType); return new CrossCompiled(SDL_GPU_SHADERFORMAT_DXBC, out.code(), out.entrypoint(), true); } if (device.supportsDxil()) { @@ -1097,7 +1165,11 @@ public final class ShaderManager { } private static void warnOnSeparateSamplers(long resources, MemoryStack stack, String stage) { - if (CrossCompileUtil.countResources(resources, Spvc.SPVC_RESOURCE_TYPE_SEPARATE_SAMPLERS, stack) > 0 && separateSamplerWarned.compareAndSet(false, true)) { + if (CrossCompileUtil.countResources(resources, Spvc.SPVC_RESOURCE_TYPE_SEPARATE_SAMPLERS, stack) > 0) warnSeparateSampler(stage); + } + + private static void warnSeparateSampler(String stage) { + if (separateSamplerWarned.compareAndSet(false, true)) { LOG.error("{} shader declares a separate sampler; SDL_GPU cannot bind one and its descriptor will be left unremapped", stage); } } @@ -1561,27 +1633,8 @@ public final class ShaderManager { } final int memberCount = Spvc.spvc_type_get_num_member_types(Spvc.spvc_compiler_get_type_handle(compiler, ubo.type_id())); for (int m = 0; m < memberCount; m++) { - final String memberName = Spvc.spvc_compiler_get_member_name(compiler, ubo.base_type_id(), m); - if (memberName == null || memberName.isEmpty()) continue; - int memberOffset = 0; - if (Spvc.spvc_compiler_type_struct_member_offset(compiler, baseTypeHandle, m, pOffset) == Spvc.SPVC_SUCCESS) { - memberOffset = pOffset.get(0); - } - int memberSize = 0; - if (Spvc.spvc_compiler_get_declared_struct_member_size(compiler, baseTypeHandle, m, pSize) == Spvc.SPVC_SUCCESS) { - memberSize = (int) pSize.get(0); - } - int memberArrayStride = 0; - if (Spvc.spvc_compiler_type_struct_member_array_stride(compiler, baseTypeHandle, m, pStride) == Spvc.SPVC_SUCCESS) { - memberArrayStride = pStride.get(0); - } - final int memberTypeId = Spvc.spvc_type_get_member_type(baseTypeHandle, m); - final long memberTypeHandle = Spvc.spvc_compiler_get_type_handle(compiler, memberTypeId); - final int memberVectorSize = Spvc.spvc_type_get_vector_size(memberTypeHandle); - final int memberColumns = Spvc.spvc_type_get_columns(memberTypeHandle); - final int memberBaseType = Spvc.spvc_type_get_basetype(memberTypeHandle); - final int memberArrayLen = (memberArrayStride > 0) ? Math.max(1, memberSize / memberArrayStride) : 1; - uboMembers.add(new UboMember(memberName, memberOffset, memberSize, memberArrayStride, memberVectorSize, memberColumns, memberBaseType, memberArrayLen)); + final UboMember member = reflectMember(compiler, ubo.base_type_id(), baseTypeHandle, m, pOffset, pStride, pSize); + if (member != null) uboMembers.add(member); } } } @@ -1722,30 +1775,35 @@ public final class ShaderManager { private static void collectBlockMembers(long compiler, SpvcReflectedResource block, long baseTypeHandle, IntBuffer pOffset, IntBuffer pStride, PointerBuffer pSize, List out) { final int memberCount = Spvc.spvc_type_get_num_member_types(Spvc.spvc_compiler_get_type_handle(compiler, block.type_id())); for (int m = 0; m < memberCount; m++) { - final String memberName = Spvc.spvc_compiler_get_member_name(compiler, block.base_type_id(), m); - if (memberName == null || memberName.isEmpty()) continue; - int memberOffset = 0; - if (Spvc.spvc_compiler_type_struct_member_offset(compiler, baseTypeHandle, m, pOffset) == Spvc.SPVC_SUCCESS) { - memberOffset = pOffset.get(0); - } - int memberSize = 0; - if (Spvc.spvc_compiler_get_declared_struct_member_size(compiler, baseTypeHandle, m, pSize) == Spvc.SPVC_SUCCESS) { - memberSize = (int) pSize.get(0); - } - int memberArrayStride = 0; - if (Spvc.spvc_compiler_type_struct_member_array_stride(compiler, baseTypeHandle, m, pStride) == Spvc.SPVC_SUCCESS) { - memberArrayStride = pStride.get(0); - } - final int memberTypeId = Spvc.spvc_type_get_member_type(baseTypeHandle, m); - final long memberTypeHandle = Spvc.spvc_compiler_get_type_handle(compiler, memberTypeId); - out.add(new UboMember(memberName, memberOffset, memberSize, memberArrayStride, - Spvc.spvc_type_get_vector_size(memberTypeHandle), - Spvc.spvc_type_get_columns(memberTypeHandle), - Spvc.spvc_type_get_basetype(memberTypeHandle), - (memberArrayStride > 0) ? Math.max(1, memberSize / memberArrayStride) : 1)); + final UboMember member = reflectMember(compiler, block.base_type_id(), baseTypeHandle, m, pOffset, pStride, pSize); + if (member != null) out.add(member); } } + private static UboMember reflectMember(long compiler, int baseTypeId, long baseTypeHandle, int m, IntBuffer pOffset, IntBuffer pStride, PointerBuffer pSize) { + final String memberName = Spvc.spvc_compiler_get_member_name(compiler, baseTypeId, m); + if (memberName == null || memberName.isEmpty()) return null; + int memberOffset = 0; + if (Spvc.spvc_compiler_type_struct_member_offset(compiler, baseTypeHandle, m, pOffset) == Spvc.SPVC_SUCCESS) { + memberOffset = pOffset.get(0); + } + int memberSize = 0; + if (Spvc.spvc_compiler_get_declared_struct_member_size(compiler, baseTypeHandle, m, pSize) == Spvc.SPVC_SUCCESS) { + memberSize = (int) pSize.get(0); + } + final long memberTypeHandle = Spvc.spvc_compiler_get_type_handle(compiler, Spvc.spvc_type_get_member_type(baseTypeHandle, m)); + int memberArrayStride = 0; + if (Spvc.spvc_type_get_num_array_dimensions(memberTypeHandle) > 0 + && Spvc.spvc_compiler_type_struct_member_array_stride(compiler, baseTypeHandle, m, pStride) == Spvc.SPVC_SUCCESS) { + memberArrayStride = pStride.get(0); + } + return new UboMember(memberName, memberOffset, memberSize, memberArrayStride, + Spvc.spvc_type_get_vector_size(memberTypeHandle), + Spvc.spvc_type_get_columns(memberTypeHandle), + Spvc.spvc_type_get_basetype(memberTypeHandle), + (memberArrayStride > 0) ? Math.max(1, memberSize / memberArrayStride) : 1); + } + private static int countResourceTypeStatic(long resources, int resourceType, MemoryStack stack) { final PointerBuffer pList = stack.pointers(0); final PointerBuffer pCount = stack.pointers(0); diff --git a/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderStageSerializer.java b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderStageSerializer.java new file mode 100644 index 000000000..0b8019bae --- /dev/null +++ b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderStageSerializer.java @@ -0,0 +1,197 @@ +package com.gtnewhorizons.angelica.sdlgpu.shader; + +import com.gtnewhorizons.angelica.glsm.shader.ShaderCacheIO; +import com.gtnewhorizons.angelica.sdlgpu.shader.ShaderManager.BlockReflection; +import com.gtnewhorizons.angelica.sdlgpu.shader.ShaderManager.FsInput; +import com.gtnewhorizons.angelica.sdlgpu.shader.ShaderManager.GraphicsBindingMap; +import com.gtnewhorizons.angelica.sdlgpu.shader.ShaderManager.PrewarmTransformResult; +import com.gtnewhorizons.angelica.sdlgpu.shader.ShaderManager.ResourceCounts; +import com.gtnewhorizons.angelica.sdlgpu.shader.ShaderManager.StageReflection; +import com.gtnewhorizons.angelica.sdlgpu.shader.ShaderManager.UboMember; +import com.gtnewhorizons.angelica.sdlgpu.shader.ShaderManager.VsInput; +import com.gtnewhorizons.angelica.sdlgpu.shader.ShaderManager.VsOutput; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import static com.gtnewhorizons.angelica.glsm.shader.ShaderCacheIO.readBytes; +import static com.gtnewhorizons.angelica.glsm.shader.ShaderCacheIO.readCount; +import static com.gtnewhorizons.angelica.glsm.shader.ShaderCacheIO.readInts; +import static com.gtnewhorizons.angelica.glsm.shader.ShaderCacheIO.readStr; +import static com.gtnewhorizons.angelica.glsm.shader.ShaderCacheIO.readStrSet; +import static com.gtnewhorizons.angelica.glsm.shader.ShaderCacheIO.writeBytes; +import static com.gtnewhorizons.angelica.glsm.shader.ShaderCacheIO.writeInts; +import static com.gtnewhorizons.angelica.glsm.shader.ShaderCacheIO.writeSortedStrs; +import static com.gtnewhorizons.angelica.glsm.shader.ShaderCacheIO.writeStr; + +final class ShaderStageSerializer { + + record Stage(byte[] spirv, StageReflection reflection, GraphicsBindingMap bindingMap, Set boolUniforms, String source) {} + + private interface Reader { + T read(ByteBuffer in); + } + + private interface Writer { + void write(DataOutputStream out, T value) throws IOException; + } + + private ShaderStageSerializer() {} + + static byte[] encode(byte[] spirv, StageReflection reflection, GraphicsBindingMap bindingMap, Set boolUniforms, String source) { + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(spirv.length + (source == null ? 0 : source.length()) + 4096); + try (DataOutputStream out = new DataOutputStream(bytes)) { + writeBytes(out, spirv); + writeReflection(out, reflection); + writeInts(out, bindingMap.roStorageTextureGlSlots()); + writeInts(out, bindingMap.rwStorageTextureGlSlots()); + writeInts(out, bindingMap.roSsboGlSlots()); + writeSortedStrs(out, boolUniforms); + writeStr(out, source); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return bytes.toByteArray(); + } + + static Stage decode(byte[] data) { + try { + final ByteBuffer in = ByteBuffer.wrap(data); + final byte[] spirv = readBytes(in); + final StageReflection reflection = readReflection(in); + final GraphicsBindingMap bindingMap = new GraphicsBindingMap(readInts(in), readInts(in), readInts(in)); + final Set bools = readStrSet(in); + final String source = readStr(in); + return new Stage(spirv, reflection, bindingMap, bools, source); + } catch (RuntimeException e) { + return null; + } + } + + static byte[] encodeTransform(String source, Set boolUniforms) { + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(source.length() + 256); + try (DataOutputStream out = new DataOutputStream(bytes)) { + writeStr(out, source); + writeSortedStrs(out, boolUniforms); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return bytes.toByteArray(); + } + + static PrewarmTransformResult decodeTransform(byte[] data) { + try { + final ByteBuffer in = ByteBuffer.wrap(data); + final String source = readStr(in); + final Set bools = readStrSet(in); + return new PrewarmTransformResult(source, bools); + } catch (RuntimeException e) { + return null; + } + } + + private static void writeReflection(DataOutputStream out, StageReflection r) throws IOException { + out.writeInt(r.counts().numSamplers()); + out.writeInt(r.counts().numUBOs()); + out.writeInt(r.counts().numStorageBuffers()); + out.writeInt(r.counts().numStorageTextures()); + writeStrList(out, r.samplerNames()); + writeStrList(out, r.extraUniformNames()); + writeStrList(out, r.storageImageNames()); + out.writeInt(r.uboSize()); + writeList(out, r.uboMembers(), ShaderStageSerializer::writeMember); + writeList(out, r.vsInputs(), (o, v) -> { + writeStr(o, v.name()); + o.writeInt(v.binaryOffset()); + o.writeInt(v.originalLocation()); + o.writeInt(v.vecSize()); + o.writeInt(v.baseType()); + }); + writeList(out, r.vsOutputs(), (o, v) -> { + writeStr(o, v.name()); + o.writeInt(v.originalLocation()); + }); + writeList(out, r.fsInputs(), (o, v) -> { + writeStr(o, v.name()); + o.writeInt(v.binaryOffset()); + o.writeInt(v.originalLocation()); + }); + out.writeInt(r.maxOutputLocation()); + out.writeInt(r.numReadonlyStorageBuffers()); + out.writeInt(r.numReadwriteStorageBuffers()); + out.writeInt(r.numReadonlyStorageTextures()); + out.writeInt(r.numReadwriteStorageTextures()); + out.writeInt(r.blocks().length); + for (BlockReflection b : r.blocks()) { + out.writeInt(b.size()); + out.writeInt(b.binding()); + writeList(out, b.members(), ShaderStageSerializer::writeMember); + out.writeBoolean(b.readOnly()); + } + } + + private static StageReflection readReflection(ByteBuffer in) { + final ResourceCounts counts = new ResourceCounts(in.getInt(), in.getInt(), in.getInt(), in.getInt()); + final List samplerNames = readStrList(in); + final List extraNames = readStrList(in); + final List storageImageNames = readStrList(in); + final int uboSize = in.getInt(); + final List uboMembers = readList(in, ShaderStageSerializer::readMember); + final List vsInputs = readList(in, i -> new VsInput(readStr(i), i.getInt(), i.getInt(), i.getInt(), i.getInt())); + final List vsOutputs = readList(in, i -> new VsOutput(readStr(i), i.getInt())); + final List fsInputs = readList(in, i -> new FsInput(readStr(i), i.getInt(), i.getInt())); + final int maxOutputLocation = in.getInt(); + final int roBuf = in.getInt(); + final int rwBuf = in.getInt(); + final int roTex = in.getInt(); + final int rwTex = in.getInt(); + final int blockCount = readCount(in); + final BlockReflection[] blocks = new BlockReflection[blockCount]; + for (int b = 0; b < blockCount; b++) { + blocks[b] = new BlockReflection(in.getInt(), in.getInt(), readList(in, ShaderStageSerializer::readMember), in.get() != 0); + } + return new StageReflection(counts, samplerNames, extraNames, storageImageNames, uboSize, uboMembers, vsInputs, vsOutputs, fsInputs, + maxOutputLocation, roBuf, rwBuf, roTex, rwTex, blocks); + } + + private static void writeMember(DataOutputStream out, UboMember m) throws IOException { + writeStr(out, m.name()); + out.writeInt(m.offset()); + out.writeInt(m.size()); + out.writeInt(m.arrayStride()); + out.writeInt(m.vectorSize()); + out.writeInt(m.columns()); + out.writeInt(m.baseType()); + out.writeInt(m.arrayLen()); + } + + private static UboMember readMember(ByteBuffer in) { + return new UboMember(readStr(in), in.getInt(), in.getInt(), in.getInt(), in.getInt(), in.getInt(), in.getInt(), in.getInt()); + } + + private static void writeList(DataOutputStream out, List list, Writer writer) throws IOException { + out.writeInt(list.size()); + for (T t : list) writer.write(out, t); + } + + private static List readList(ByteBuffer in, Reader reader) { + final int n = readCount(in); + final List out = new ArrayList<>(n); + for (int i = 0; i < n; i++) out.add(reader.read(in)); + return out; + } + + private static void writeStrList(DataOutputStream out, List list) throws IOException { + writeList(out, list, ShaderCacheIO::writeStr); + } + + private static List readStrList(ByteBuffer in) { + return readList(in, ShaderCacheIO::readStr); + } +} diff --git a/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/cross/CrossCompileCache.java b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/cross/CrossCompileCache.java new file mode 100644 index 000000000..756e6ff63 --- /dev/null +++ b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/cross/CrossCompileCache.java @@ -0,0 +1,81 @@ +package com.gtnewhorizons.angelica.sdlgpu.shader.cross; + +import com.gtnewhorizons.angelica.glsm.shader.ShaderCacheIO; +import com.gtnewhorizons.angelica.glsm.shader.ShaderDiskCache; +import it.unimi.dsi.fastutil.objects.Object2ObjectLinkedOpenHashMap; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.function.Supplier; + +public final class CrossCompileCache { + + public record Output(ByteBuffer code, String entrypoint) {} + + @FunctionalInterface + public interface Uncached { + Output compile(ByteBuffer spirv, int glShaderType); + } + + private static final int CACHE_MAX = 256; + + private static final class CacheKey { + private final byte[] spirvBytes; + private final int glShaderType; + private final int hash; + + CacheKey(byte[] b, int t) { + this.spirvBytes = b; + this.glShaderType = t; + this.hash = Arrays.hashCode(b) * 31 + t; + } + + @Override public int hashCode() { return hash; } + + @Override public boolean equals(Object o) { + return o instanceof CacheKey k && k.glShaderType == glShaderType && k.hash == hash && Arrays.equals(k.spirvBytes, spirvBytes); + } + } + + private record CacheValue(byte[] codeBytes, String entrypoint) {} + + private final Object2ObjectLinkedOpenHashMap cache = new Object2ObjectLinkedOpenHashMap<>(); + private final String layer; + private final Supplier toolchainId; + private final Uncached uncached; + + public CrossCompileCache(String layer, Supplier toolchainId, Uncached uncached) { + this.layer = layer; + this.toolchainId = toolchainId; + this.uncached = uncached; + } + + public Output compile(ByteBuffer spirv, int glShaderType) { + final byte[] spirvHeap = ShaderCacheIO.toHeap(spirv); + final CacheKey key = new CacheKey(spirvHeap, glShaderType); + final CacheValue cached; + synchronized (cache) { + cached = cache.getAndMoveToFirst(key); + } + if (cached != null) return new Output(ShaderCacheIO.toNative(cached.codeBytes), cached.entrypoint); + + final ShaderDiskCache.Key diskKey = ShaderDiskCache.isEnabled() ? diskKey(spirvHeap, glShaderType) : null; + if (diskKey != null) { + final ShaderDiskCache.Blob blob = ShaderDiskCache.getBlob(diskKey); + if (blob != null) { + ShaderCacheIO.lruPut(cache, key, new CacheValue(blob.data(), blob.tag()), CACHE_MAX); + return new Output(ShaderCacheIO.toNative(blob.data()), blob.tag()); + } + } + + final Output out = uncached.compile(spirv, glShaderType); + final byte[] codeHeap = ShaderCacheIO.toHeap(out.code()); + ShaderCacheIO.lruPut(cache, key, new CacheValue(codeHeap, out.entrypoint()), CACHE_MAX); + if (diskKey != null) ShaderDiskCache.putBlob(diskKey, out.entrypoint(), codeHeap); + return out; + } + + public ShaderDiskCache.Key diskKey(byte[] spirvHeap, int glShaderType) { + return ShaderDiskCache.key(layer).str(toolchainId.get()).bytes(spirvHeap).i(glShaderType); + } +} diff --git a/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/cross/CrossCompileUtil.java b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/cross/CrossCompileUtil.java index 717853f14..db0d01d9d 100644 --- a/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/cross/CrossCompileUtil.java +++ b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/cross/CrossCompileUtil.java @@ -2,6 +2,7 @@ package com.gtnewhorizons.angelica.sdlgpu.shader.cross; import org.lwjgl.system.MemoryUtil; import com.gtnewhorizons.angelica.config.SystemProperties; +import com.gtnewhorizons.angelica.glsm.shader.ShaderCacheIO; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.lwjgl.PointerBuffer; @@ -27,8 +28,16 @@ public final class CrossCompileUtil { public static final AtomicInteger SHADER_DUMP_COUNTER = new AtomicInteger(); + private static final class SpvcId { + static final String VALUE = ShaderCacheIO.libraryId(Spvc.getLibrary()); + } + private CrossCompileUtil() {} + public static String spvcId() { + return SpvcId.VALUE; + } + public static String stageSuffix(int glShaderType) { if (glShaderType == GL20.GL_VERTEX_SHADER) return "vert"; if (glShaderType == GL20.GL_FRAGMENT_SHADER) return "frag"; diff --git a/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/dxbc/D3DCompiler.java b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/dxbc/D3DCompiler.java index be874bc1e..e846363ae 100644 --- a/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/dxbc/D3DCompiler.java +++ b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/dxbc/D3DCompiler.java @@ -1,5 +1,6 @@ package com.gtnewhorizons.angelica.sdlgpu.shader.dxbc; +import com.gtnewhorizons.angelica.glsm.shader.ShaderCacheIO; import org.lwjgl.PointerBuffer; import org.lwjgl.system.JNI; import org.lwjgl.system.Library; @@ -34,6 +35,8 @@ public final class D3DCompiler { private static final SharedLibrary D3DCOMPILER = Library.loadNative(D3DCompiler.class, "com.gtnewhorizons.angelica.sdlgpu", null, "d3dcompiler_47", "d3dcompiler"); + public static final String LIBRARY_ID = ShaderCacheIO.libraryId(D3DCOMPILER); + private static final long D3D_COMPILE_ADDR = apiGetFunctionAddress(D3DCOMPILER, "D3DCompile"); public static final int D3DCOMPILE_ENABLE_STRICTNESS = 0x800; diff --git a/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/dxbc/DxbcCrossCompile.java b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/dxbc/DxbcCrossCompile.java index 7daed2fac..5b416012d 100644 --- a/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/dxbc/DxbcCrossCompile.java +++ b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/dxbc/DxbcCrossCompile.java @@ -2,8 +2,9 @@ package com.gtnewhorizons.angelica.sdlgpu.shader.dxbc; import org.lwjgl.system.MemoryUtil; import com.gtnewhorizons.angelica.config.SystemProperties; +import com.gtnewhorizons.angelica.sdlgpu.shader.cross.CrossCompileCache; +import com.gtnewhorizons.angelica.sdlgpu.shader.cross.CrossCompileCache.Output; import com.gtnewhorizons.angelica.sdlgpu.shader.cross.CrossCompileUtil; -import it.unimi.dsi.fastutil.objects.Object2ObjectLinkedOpenHashMap; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.lwjgl.PointerBuffer; @@ -17,11 +18,9 @@ import org.lwjgl.util.spvc.SpvcReflectedResource; import java.nio.ByteBuffer; import java.nio.IntBuffer; -import java.util.Arrays; import static org.lwjgl.system.MemoryStack.stackPush; import static org.lwjgl.system.MemoryUtil.memAddress; -import static org.lwjgl.system.MemoryUtil.memAlloc; import static org.lwjgl.system.MemoryUtil.memByteBufferNT1; import static org.lwjgl.system.MemoryUtil.memUTF8; @@ -30,56 +29,23 @@ public final class DxbcCrossCompile { private static final Logger LOG = LogManager.getLogger("Angelica-SDLGPU"); private static final String BACKEND = "DXBC"; - public record Output(ByteBuffer code, String entrypoint) {} - - private static final int CACHE_MAX = 256; - private static final Object2ObjectLinkedOpenHashMap CACHE = new Object2ObjectLinkedOpenHashMap<>(); - - private static final class CacheKey { - public final byte[] spirvBytes; - public final int glShaderType; - public final int hash; - public CacheKey(byte[] b, int t) { - this.spirvBytes = b; - this.glShaderType = t; - this.hash = Arrays.hashCode(b) * 31 + t; - } - @Override public int hashCode() { return hash; } - @Override public boolean equals(Object o) { - if (!(o instanceof CacheKey k)) return false; - return k.glShaderType == glShaderType && k.hash == hash && Arrays.equals(k.spirvBytes, spirvBytes); - } - } - - private record CacheValue(byte[] codeBytes, String entrypoint) {} - - public static void clearCache() { - synchronized (CACHE) { CACHE.clear(); } - } + private static final CrossCompileCache CACHE = new CrossCompileCache("dxbc", () -> CrossCompileUtil.spvcId() + '|' + D3DCompiler.LIBRARY_ID, DxbcCrossCompile::crossCompileUncached); private DxbcCrossCompile() {} public static Output compile(ByteBuffer spirv, int glShaderType) { - final boolean isCompute = (glShaderType == GL43.GL_COMPUTE_SHADER); - final String target; - if (glShaderType == GL20.GL_VERTEX_SHADER) target = "vs_5_1"; - else if (glShaderType == GL20.GL_FRAGMENT_SHADER) target = "ps_5_1"; - else if (isCompute) target = "cs_5_1"; - else throw new UnsupportedOperationException("Unsupported shader type for DXBC cross-compile: 0x" + Integer.toHexString(glShaderType)); + return CACHE.compile(spirv, glShaderType); + } - final byte[] spirvHeap = new byte[spirv.remaining()]; - spirv.duplicate().get(spirvHeap); - final CacheKey key = new CacheKey(spirvHeap, glShaderType); - final CacheValue cached; - synchronized (CACHE) { - cached = CACHE.getAndMoveToFirst(key); - } - if (cached != null) { - final ByteBuffer copy = memAlloc(cached.codeBytes.length); - copy.put(cached.codeBytes).flip(); - return new Output(copy, cached.entrypoint); - } + private static String targetFor(int glShaderType) { + if (glShaderType == GL20.GL_VERTEX_SHADER) return "vs_5_1"; + if (glShaderType == GL20.GL_FRAGMENT_SHADER) return "ps_5_1"; + if (glShaderType == GL43.GL_COMPUTE_SHADER) return "cs_5_1"; + throw new UnsupportedOperationException("Unsupported shader type for DXBC cross-compile: 0x" + Integer.toHexString(glShaderType)); + } + private static Output crossCompileUncached(ByteBuffer spirv, int glShaderType) { + final String target = targetFor(glShaderType); final int dumpId = SystemProperties.dumpShaders() ? CrossCompileUtil.SHADER_DUMP_COUNTER.getAndIncrement() : -1; if (dumpId >= 0) CrossCompileUtil.dumpSpirv(spirv, dumpId, glShaderType); @@ -90,12 +56,6 @@ public final class DxbcCrossCompile { try { final ByteBuffer dxbc = D3DCompiler.compile(memAddress(hlslNative), hlslNative.remaining(), "main", target, D3DCompiler.D3DCOMPILE_OPTIMIZATION_LEVEL3 | D3DCompiler.D3DCOMPILE_ENABLE_STRICTNESS, 0); if (dumpId >= 0) CrossCompileUtil.dumpBytes(dxbc, dumpId, glShaderType, "dxbc"); - final byte[] dxbcHeap = new byte[dxbc.remaining()]; - dxbc.duplicate().get(dxbcHeap); - synchronized (CACHE) { - CACHE.putAndMoveToFirst(key, new CacheValue(dxbcHeap, "main")); - while (CACHE.size() > CACHE_MAX) CACHE.removeLast(); - } return new Output(dxbc, "main"); } finally { MemoryUtil.memFree(hlslNative); diff --git a/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/msl/MslCrossCompile.java b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/msl/MslCrossCompile.java index 07958a3f2..71808ccd6 100644 --- a/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/msl/MslCrossCompile.java +++ b/sdl-gpu/src/main/java/com/gtnewhorizons/angelica/sdlgpu/shader/msl/MslCrossCompile.java @@ -1,8 +1,9 @@ package com.gtnewhorizons.angelica.sdlgpu.shader.msl; import com.gtnewhorizons.angelica.config.SystemProperties; +import com.gtnewhorizons.angelica.sdlgpu.shader.cross.CrossCompileCache; +import com.gtnewhorizons.angelica.sdlgpu.shader.cross.CrossCompileCache.Output; import com.gtnewhorizons.angelica.sdlgpu.shader.cross.CrossCompileUtil; -import it.unimi.dsi.fastutil.objects.Object2ObjectLinkedOpenHashMap; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.lwjgl.PointerBuffer; @@ -16,10 +17,8 @@ import org.lwjgl.util.spvc.SpvcReflectedResource; import java.nio.ByteBuffer; import java.nio.IntBuffer; -import java.util.Arrays; import static org.lwjgl.system.MemoryStack.stackPush; -import static org.lwjgl.system.MemoryUtil.memAlloc; import static org.lwjgl.system.MemoryUtil.memUTF8; public final class MslCrossCompile { @@ -27,32 +26,7 @@ public final class MslCrossCompile { private static final Logger LOG = LogManager.getLogger("Angelica-SDLGPU"); private static final String BACKEND = "MSL"; - public record Output(ByteBuffer code, String entrypoint) {} - - private static final int CACHE_MAX = 256; - private static final Object2ObjectLinkedOpenHashMap CACHE = new Object2ObjectLinkedOpenHashMap<>(); - - private static final class CacheKey { - public final byte[] spirvBytes; - public final int glShaderType; - public final int hash; - public CacheKey(byte[] b, int t) { - this.spirvBytes = b; - this.glShaderType = t; - this.hash = Arrays.hashCode(b) * 31 + t; - } - @Override public int hashCode() { return hash; } - @Override public boolean equals(Object o) { - if (!(o instanceof CacheKey k)) return false; - return k.glShaderType == glShaderType && k.hash == hash && Arrays.equals(k.spirvBytes, spirvBytes); - } - } - - private record CacheValue(byte[] codeBytes, String entrypoint) {} - - public static void clearCache() { - synchronized (CACHE) { CACHE.clear(); } - } + private static final CrossCompileCache CACHE = new CrossCompileCache("msl", CrossCompileUtil::spvcId, MslCrossCompile::crossCompileUncached); @FunctionalInterface private interface BufferSlotFn { public int slot(int set, int binding); } @@ -63,30 +37,18 @@ public final class MslCrossCompile { private MslCrossCompile() {} public static Output compile(ByteBuffer spirv, int glShaderType) { - final int executionModel; - if (glShaderType == GL20.GL_VERTEX_SHADER) { - executionModel = Spv.SpvExecutionModelVertex; - } else if (glShaderType == GL20.GL_FRAGMENT_SHADER) { - executionModel = Spv.SpvExecutionModelFragment; - } else if (glShaderType == GL43.GL_COMPUTE_SHADER) { - executionModel = Spv.SpvExecutionModelGLCompute; - } else { - throw new UnsupportedOperationException("Unsupported shader type for MSL cross-compile: 0x" + Integer.toHexString(glShaderType)); - } + return CACHE.compile(spirv, glShaderType); + } - final byte[] spirvHeap = new byte[spirv.remaining()]; - spirv.duplicate().get(spirvHeap); - final CacheKey key = new CacheKey(spirvHeap, glShaderType); - final CacheValue cached; - synchronized (CACHE) { - cached = CACHE.getAndMoveToFirst(key); - } - if (cached != null) { - final ByteBuffer copy = memAlloc(cached.codeBytes.length); - copy.put(cached.codeBytes).flip(); - return new Output(copy, cached.entrypoint); - } + private static int executionModelFor(int glShaderType) { + if (glShaderType == GL20.GL_VERTEX_SHADER) return Spv.SpvExecutionModelVertex; + if (glShaderType == GL20.GL_FRAGMENT_SHADER) return Spv.SpvExecutionModelFragment; + if (glShaderType == GL43.GL_COMPUTE_SHADER) return Spv.SpvExecutionModelGLCompute; + throw new UnsupportedOperationException("Unsupported shader type for MSL cross-compile: 0x" + Integer.toHexString(glShaderType)); + } + private static Output crossCompileUncached(ByteBuffer spirv, int glShaderType) { + final int executionModel = executionModelFor(glShaderType); final int dumpId = SystemProperties.dumpShaders() ? CrossCompileUtil.SHADER_DUMP_COUNTER.getAndIncrement() : -1; if (dumpId >= 0) CrossCompileUtil.dumpSpirv(spirv, dumpId, glShaderType); @@ -142,12 +104,6 @@ public final class MslCrossCompile { final ByteBuffer code = memUTF8(mslSrc, true); final String rawEntry = Spvc.spvc_compiler_get_cleansed_entry_point_name(compiler, "main", executionModel); final String entry = rawEntry != null ? rawEntry : "main0"; - final byte[] codeHeap = new byte[code.remaining()]; - code.duplicate().get(codeHeap); - synchronized (CACHE) { - CACHE.putAndMoveToFirst(key, new CacheValue(codeHeap, entry)); - while (CACHE.size() > CACHE_MAX) CACHE.removeLast(); - } return new Output(code, entry); } finally { Spvc.spvc_context_destroy(ctx); diff --git a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/FontFilterVertexSamplerTest.java b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/FontFilterVertexSamplerTest.java index 08bb19f38..f3ce6aa37 100644 --- a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/FontFilterVertexSamplerTest.java +++ b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/FontFilterVertexSamplerTest.java @@ -24,8 +24,8 @@ class FontFilterVertexSamplerTest { private static String runSdlPipeline(String raw) { final GlslVulkanPreprocess.Result pre = GlslVulkanPreprocess.run(raw, GL20.GL_VERTEX_SHADER, "fontFilter.vsh", true); String src = pre != null ? pre.rewrittenSource() : raw; - src = ClipZRemap.injectGLToVulkanClipZ(src); - src = SamplerStripper.stripUnused(src); + src = ShaderTransformChain.clipZ(src); + src = ShaderTransformChain.stripUnused(src); return src; } diff --git a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/PerFrameBlockInjectorTest.java b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/PerFrameBlockInjectorTest.java index 21e323dbb..73b979edd 100644 --- a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/PerFrameBlockInjectorTest.java +++ b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/PerFrameBlockInjectorTest.java @@ -27,7 +27,7 @@ class PerFrameBlockInjectorTest { @Test void dropsLooseDeclarationsAndInjectsTheBlock() { - final String out = PerFrameBlockInjector.inject(""" + final String out = ShaderTransformChain.inject(""" #version 330 core uniform vec3 cameraPosition; uniform float rainStrength; @@ -43,7 +43,7 @@ class PerFrameBlockInjectorTest { @Test void declaresEveryMemberEvenWhenUnused() { - final String out = PerFrameBlockInjector.inject(""" + final String out = ShaderTransformChain.inject(""" #version 330 core uniform float rainStrength; out vec4 fragColor; @@ -57,7 +57,7 @@ class PerFrameBlockInjectorTest { @Test void blockGoesAfterTheVersionDirective() { - final String out = PerFrameBlockInjector.inject(""" + final String out = ShaderTransformChain.inject(""" #version 330 core uniform float rainStrength; void main() {} @@ -67,7 +67,7 @@ class PerFrameBlockInjectorTest { @Test void leavesMixedDeclarationAlone() { - final String out = PerFrameBlockInjector.inject(""" + final String out = ShaderTransformChain.inject(""" #version 330 core uniform float rainStrength, packOwnedValue; void main() {} @@ -78,19 +78,19 @@ class PerFrameBlockInjectorTest { @Test void emptyMemberListIsANoOp() { final String src = "#version 330 core\nvoid main() {}\n"; - assertEquals(src, PerFrameBlockInjector.inject(src, new PerFrameUniformBlock(List.of()), null)); + assertEquals(src, ShaderTransformChain.inject(src, new PerFrameUniformBlock(List.of()), null)); } @Test void noBlocksReturnsTheSourceInstance() { final String src = "#version 330 core\nuniform float frameTimeCounter;\nvoid main() {}\n"; - assertSame(src, PerFrameBlockInjector.inject(src, null, null)); - assertSame(src, PerFrameBlockInjector.inject(src, new PerFrameUniformBlock(List.of()), new PerFrameUniformBlock(List.of()))); + assertSame(src, ShaderTransformChain.inject(src, null, null)); + assertSame(src, ShaderTransformChain.inject(src, new PerFrameUniformBlock(List.of()), new PerFrameUniformBlock(List.of()))); } @Test void memberShadowedByAPackDeclarationIsRenamedNotRedeclared() { - final String out = PerFrameBlockInjector.inject(""" + final String out = ShaderTransformChain.inject(""" #version 330 core const float rainStrength = 3.14159265359; out vec4 fragColor; @@ -108,7 +108,7 @@ class PerFrameBlockInjectorTest { @Test void functionLocalRedeclarationDoesNotShadowTheMember() { - final String out = PerFrameBlockInjector.inject(""" + final String out = ShaderTransformChain.inject(""" #version 330 core uniform vec3 cameraPosition; vec3 ambient = pow(cameraPosition, vec3(0.75)); @@ -131,12 +131,12 @@ class PerFrameBlockInjectorTest { @Test void shadowingDoesNotChangeAnyOffset() { - final String plain = PerFrameBlockInjector.inject(""" + final String plain = ShaderTransformChain.inject(""" #version 330 core out vec4 fragColor; void main() { fragColor = vec4(cameraPosition, rainStrength); } """, BLOCK, null); - final String shadowed = PerFrameBlockInjector.inject(""" + final String shadowed = ShaderTransformChain.inject(""" #version 330 core const float rainStrength = 1.0; out vec4 fragColor; @@ -152,7 +152,7 @@ class PerFrameBlockInjectorTest { new Member("cameraPosition", UniformType.VEC3), new Member("isRightHanded", UniformType.INT))); - final String out = PerFrameBlockInjector.inject(""" + final String out = ShaderTransformChain.inject(""" #version 330 core uniform bool isRightHanded; out vec4 fragColor; @@ -170,7 +170,7 @@ class PerFrameBlockInjectorTest { @Test void uniformDeclaredWithTheMatchingTypeIsStillReplaced() { - final String out = PerFrameBlockInjector.inject(""" + final String out = ShaderTransformChain.inject(""" #version 330 core uniform float rainStrength; out vec4 fragColor; @@ -188,12 +188,12 @@ class PerFrameBlockInjectorTest { out vec4 fragColor; void main() { fragColor = packOwnedColor; } """; - assertEquals(src, PerFrameBlockInjector.inject(src, BLOCK, null)); + assertEquals(src, ShaderTransformChain.inject(src, BLOCK, null)); } @Test void oneReferenceStillDeclaresEveryMember() { - final String out = PerFrameBlockInjector.inject(""" + final String out = ShaderTransformChain.inject(""" #version 330 core out vec4 fragColor; void main() { fragColor = vec4(rainStrength); } @@ -209,7 +209,7 @@ class PerFrameBlockInjectorTest { @Test void bothBlocksInjectInOneParse() { - final String out = PerFrameBlockInjector.inject(""" + final String out = ShaderTransformChain.inject(""" #version 330 core uniform vec3 cameraPosition; uniform mat4 gbufferModelView; @@ -226,7 +226,7 @@ class PerFrameBlockInjectorTest { @Test void unreferencedBlockIsNotInjected() { - final String out = PerFrameBlockInjector.inject(""" + final String out = ShaderTransformChain.inject(""" #version 330 core out vec4 fragColor; void main() { fragColor = vec4(cameraPosition, 1.0); } @@ -253,7 +253,7 @@ class PerFrameBlockInjectorTest { @Test void injectedSourceCompilesAndReflects() { - final String out = PerFrameBlockInjector.inject(""" + final String out = ShaderTransformChain.inject(""" #version 330 core uniform vec3 cameraPosition; uniform float rainStrength; diff --git a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/PerFrameBlockLocationRoutingTest.java b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/PerFrameBlockLocationRoutingTest.java index 0fab66db3..0e536efb7 100644 --- a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/PerFrameBlockLocationRoutingTest.java +++ b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/PerFrameBlockLocationRoutingTest.java @@ -38,7 +38,7 @@ class PerFrameBlockLocationRoutingTest { """; private static ShaderManager.ProgramObject linkedProgram() { - final String injected = PerFrameBlockInjector.inject(SOURCE, PER_FRAME, PER_PASS); + final String injected = ShaderTransformChain.inject(SOURCE, PER_FRAME, PER_PASS); final SpirvCompiler.Result r = SpirvCompiler.compile(injected, Shaderc.shaderc_fragment_shader, "routing.frag", SpirvCompiler.Options.vulkanForced460Core()); assertNotNull(r.spirv(), () -> "compile failed: " + r.error()); diff --git a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/PerFrameBlockReadOnlyTest.java b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/PerFrameBlockReadOnlyTest.java index 28fd496ca..20e4570fc 100644 --- a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/PerFrameBlockReadOnlyTest.java +++ b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/PerFrameBlockReadOnlyTest.java @@ -30,7 +30,7 @@ class PerFrameBlockReadOnlyTest { @Test void injectedBlocksAreDecoratedNonWritable() { - final String injected = PerFrameBlockInjector.inject(DUAL_SOURCE, BLOCK, PER_PASS); + final String injected = ShaderTransformChain.inject(DUAL_SOURCE, BLOCK, PER_PASS); final SpirvCompiler.Result r = SpirvCompiler.compile(injected, Shaderc.shaderc_fragment_shader, "readonly.frag", SpirvCompiler.Options.vulkanForced460Core()); assertNotNull(r.spirv(), () -> "compile failed: " + r.error()); diff --git a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/PrewarmSingleParseEquivalenceTest.java b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/PrewarmSingleParseEquivalenceTest.java index c86738bb3..c51a50d87 100644 --- a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/PrewarmSingleParseEquivalenceTest.java +++ b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/PrewarmSingleParseEquivalenceTest.java @@ -27,7 +27,7 @@ class PrewarmSingleParseEquivalenceTest { GlslVulkanPreprocess.clearCache(); final String legacyOut = ShaderTransformChain.run(source, glShaderType); GlslVulkanPreprocess.clearCache(); - final String parseOnceOut = ShaderManager.applyPrewarmTransforms(source, glShaderType); + final String parseOnceOut = ShaderManager.applyPrewarmTransformsFull(source, glShaderType, GLSMHooks.perFrameUniformBlock, GLSMHooks.perPassUniformBlock).source(); assertEquals(legacyOut, parseOnceOut, "single-parse vs chained pipeline diverged for " + label); } @@ -35,7 +35,7 @@ class PrewarmSingleParseEquivalenceTest { GlslVulkanPreprocess.clearCache(); final String legacyOut = ShaderTransformChain.run(source, glShaderType); GlslVulkanPreprocess.clearCache(); - final String parseOnceOut = ShaderManager.applyPrewarmTransforms(source, glShaderType); + final String parseOnceOut = ShaderManager.applyPrewarmTransformsFull(source, glShaderType, GLSMHooks.perFrameUniformBlock, GLSMHooks.perPassUniformBlock).source(); assertEquals(legacyOut.replaceAll("\\s+", " "), parseOnceOut.replaceAll("\\s+", " "), "single-parse vs chained pipeline diverged beyond whitespace for " + label); } @@ -141,6 +141,29 @@ class PrewarmSingleParseEquivalenceTest { assertEquivalent("geometry-passthrough", src, GL32.GL_GEOMETRY_SHADER); } + @Test + void readOnlyImage_rewrittenToTextureWithSamplerlessExtension() { + final String src = "#version 460 core\n" + + "layout(binding = 2, rgba8) uniform readonly image2D u_Img;\n" + + "layout(location = 0) out vec4 fragColor;\n" + + "void main() {\n" + + " fragColor = imageLoad(u_Img, ivec2(1, 2)) + vec4(imageSize(u_Img), 0.0, 1.0);\n" + + "}\n"; + assertEquivalent("readonly-image", src, GL20.GL_FRAGMENT_SHADER); + } + + @Test + void samplerlessTexture_extensionInserted() { + final String src = "#version 460 core\n" + + "uniform texture2D u_Tex;\n" + + "uniform sampler u_Smp;\n" + + "layout(location = 0) out vec4 fragColor;\n" + + "void main() {\n" + + " fragColor = texelFetch(u_Tex, ivec2(0), 0) + texture(sampler2D(u_Tex, u_Smp), vec2(0.5));\n" + + "}\n"; + assertEquivalent("samplerless-texture", src, GL20.GL_FRAGMENT_SHADER); + } + @Test void unparseable_fallsBackToInputUnchanged() { final String src = "this is not valid GLSL @@@@\n"; diff --git a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/PrewarmTreeHandoffEquivalenceTest.java b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/PrewarmTreeHandoffEquivalenceTest.java index 503aa3a06..84a5b9276 100644 --- a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/PrewarmTreeHandoffEquivalenceTest.java +++ b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/PrewarmTreeHandoffEquivalenceTest.java @@ -34,9 +34,9 @@ class PrewarmTreeHandoffEquivalenceTest { final String finalSource = HEADER + printedBody; GlslVulkanPreprocess.clearCache(); - final ShaderManager.PrewarmTransformResult reparse = ShaderManager.applyPrewarmTransformsFull(finalSource, glShaderType); + final ShaderManager.PrewarmTransformResult reparse = ShaderManager.applyPrewarmTransformsFull(finalSource, glShaderType, GLSMHooks.perFrameUniformBlock, GLSMHooks.perPassUniformBlock); GlslVulkanPreprocess.clearCache(); - final ShaderManager.PrewarmTransformResult treeFed = ShaderManager.applyPrewarmTransformsFull(finalSource, tree, HEADER.length(), glShaderType); + final ShaderManager.PrewarmTransformResult treeFed = ShaderManager.applyPrewarmTransformsFull(finalSource, tree, HEADER.length(), glShaderType, GLSMHooks.perFrameUniformBlock, GLSMHooks.perPassUniformBlock); assertEquals(reparse.source(), treeFed.source(), "tree handoff diverged from re-parse for " + label); assertEquals(reparse.boolUniforms(), treeFed.boolUniforms(), "bool uniforms diverged for " + label); diff --git a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/SdlSamplerStripperPreprocessorTest.java b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/SdlSamplerStripperPreprocessorTest.java index 36fe6e4e2..1e99c2c7f 100644 --- a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/SdlSamplerStripperPreprocessorTest.java +++ b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/SdlSamplerStripperPreprocessorTest.java @@ -18,7 +18,7 @@ class SdlSamplerStripperPreprocessorTest { + " float a = textureLod(u_S, vec2(0), 0.0).r;\n" + "#endif\n" + "}\n"; - final String out = SamplerStripper.stripUnused(src); + final String out = ShaderTransformChain.stripUnused(src); assertTrue(out.contains("uniform sampler2D u_S"), "ref inside #if must keep the decl; output:\n" + out); } @@ -33,7 +33,7 @@ class SdlSamplerStripperPreprocessorTest { + " float a = t * 2.0;\n" + "#endif\n" + "}\n"; - final String out = SamplerStripper.stripUnused(src); + final String out = ShaderTransformChain.stripUnused(src); assertTrue(out.contains("uniform sampler2D u_S"), "unconditional ref must keep the decl; output:\n" + out); } @@ -50,7 +50,7 @@ class SdlSamplerStripperPreprocessorTest { + " float b = textureLod(u_S, vec2(0), 0.0).r;\n" + "#endif\n" + "}\n"; - final String out = SamplerStripper.stripUnused(src); + final String out = ShaderTransformChain.stripUnused(src); assertTrue(out.contains("uniform sampler2D u_S"), "refs in both branches must keep the decl; output:\n" + out); } @@ -72,7 +72,7 @@ class SdlSamplerStripperPreprocessorTest { + " fragColor = texture(u_BlockTex, v_TexCoord);\n" + "#endif\n" + "}\n"; - final String out = SamplerStripper.stripUnused(src); + final String out = ShaderTransformChain.stripUnused(src); assertTrue(out.contains("uniform sampler2D u_BlockTex"), "terrain sampler must survive; output:\n" + out); } @@ -86,7 +86,7 @@ class SdlSamplerStripperPreprocessorTest { + " float a = 1.0;\n" + "#endif\n" + "}\n"; - final String out = SamplerStripper.stripUnused(src); + final String out = ShaderTransformChain.stripUnused(src); assertFalse(out.contains("uniform sampler2D iris_centerDepthSmooth"), "an unreferenced sampler must still be stripped; output:\n" + out); } @@ -101,7 +101,7 @@ class SdlSamplerStripperPreprocessorTest { + " float a = 1.0;\n" + "#endif\n" + "}\n"; - final String out = SamplerStripper.stripUnused(src); + final String out = ShaderTransformChain.stripUnused(src); assertFalse(out.contains("uniform sampler2D u_S"), "a comment mention must not keep the decl; output:\n" + out); } @@ -112,7 +112,7 @@ class SdlSamplerStripperPreprocessorTest { + "void main() {\n" + " gl_Position = vec4(v_Position, 1.0);\n" + "}\n"; - assertSame(src, SamplerStripper.stripUnused(src)); + assertSame(src, ShaderTransformChain.stripUnused(src)); } @Test @@ -123,7 +123,7 @@ class SdlSamplerStripperPreprocessorTest { + "void main() {\n" + " float a = SAMPLE_IT();\n" + "}\n"; - final String out = SamplerStripper.stripUnused(src); + final String out = ShaderTransformChain.stripUnused(src); assertTrue(out.contains("uniform sampler2D u_S"), "a macro-body ref must keep the decl; output:\n" + out); } } diff --git a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderSourceTransformCacheTest.java b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderSourceTransformCacheTest.java index d06a94ba8..884046f60 100644 --- a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderSourceTransformCacheTest.java +++ b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderSourceTransformCacheTest.java @@ -66,8 +66,8 @@ class ShaderSourceTransformCacheTest { } @Test - void repeatedSourceReusesTheChainedTransform() { - final String expected = ShaderTransformChain.run(FRAG, GL20.GL_FRAGMENT_SHADER); + void repeatedSourceReusesTheTransform() { + final String expected = ShaderManager.applyPrewarmTransformsFull(FRAG, GL20.GL_FRAGMENT_SHADER, GLSMHooks.perFrameUniformBlock, GLSMHooks.perPassUniformBlock).source(); GlslVulkanPreprocess.clearCache(); final ShaderManager sm = new ShaderManager(null); @@ -81,8 +81,8 @@ class ShaderSourceTransformCacheTest { } @Test - void vertexSourceMatchesTheChainAndCarriesClipZ() { - final String expected = ShaderTransformChain.run(VERT, GL20.GL_VERTEX_SHADER); + void vertexSourceMatchesTheSingleParseAndCarriesClipZ() { + final String expected = ShaderManager.applyPrewarmTransformsFull(VERT, GL20.GL_VERTEX_SHADER, GLSMHooks.perFrameUniformBlock, GLSMHooks.perPassUniformBlock).source(); GlslVulkanPreprocess.clearCache(); final ShaderManager sm = new ShaderManager(null); @@ -104,7 +104,7 @@ class ShaderSourceTransformCacheTest { GLSMHooks.perFrameUniformBlock = null; GlslVulkanPreprocess.clearCache(); - final String expected = ShaderTransformChain.run(PFB_FRAG, GL20.GL_FRAGMENT_SHADER); + final String expected = ShaderManager.applyPrewarmTransformsFull(PFB_FRAG, GL20.GL_FRAGMENT_SHADER, GLSMHooks.perFrameUniformBlock, GLSMHooks.perPassUniformBlock).source(); GlslVulkanPreprocess.clearCache(); final int withoutBlock = sm.createShader(GL20.GL_FRAGMENT_SHADER); diff --git a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderStageSerializerTest.java b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderStageSerializerTest.java new file mode 100644 index 000000000..c9aa76b73 --- /dev/null +++ b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderStageSerializerTest.java @@ -0,0 +1,207 @@ +package com.gtnewhorizons.angelica.sdlgpu.shader; + +import com.gtnewhorizons.angelica.glsm.hooks.PerFrameUniformBlock; +import com.gtnewhorizons.angelica.glsm.hooks.PerFrameUniformBlock.Member; +import com.gtnewhorizons.angelica.glsm.shader.ShaderCacheIO; +import com.gtnewhorizons.angelica.glsm.shader.SpirvCompiler; +import com.gtnewhorizons.angelica.glsm.shader.UniformType; +import org.junit.jupiter.api.Test; +import org.lwjgl.opengl.GL20; +import org.lwjgl.util.shaderc.Shaderc; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.lwjgl.system.MemoryUtil.memFree; + +class ShaderStageSerializerTest { + + private static final PerFrameUniformBlock BLOCK = new PerFrameUniformBlock(List.of( + new Member("cameraPosition", UniformType.VEC3), + new Member("rainStrength", UniformType.FLOAT))); + + private static final String VERTEX = """ + #version 460 core + layout(location = 0) in vec3 a_Position; + layout(location = 1) in vec2 a_TexCoord; + layout(location = 0) out vec2 v_TexCoord; + layout(location = 1) out vec3 v_Color; + uniform vec3 cameraPosition; + uniform float rainStrength; + layout(std140, binding = 0) uniform Scene { mat4 u_Mvp; vec4 u_Tints[3]; float u_Scale; } scene; + void main() { + v_TexCoord = a_TexCoord; + v_Color = cameraPosition * rainStrength * scene.u_Tints[1].xyz * scene.u_Scale; + gl_Position = scene.u_Mvp * vec4(a_Position, 1.0); + } + """; + + private static final String FRAGMENT = """ + #version 460 core + layout(binding = 0) uniform sampler2D u_TexA; + layout(binding = 1) uniform sampler2D u_TexB; + layout(binding = 2, rgba8) readonly uniform image2D u_Img; + layout(binding = 3) uniform sampler u_Smp; + layout(binding = 4) uniform texture2D u_Tex; + layout(location = 0) in vec2 v_TexCoord; + layout(location = 1) in vec3 v_Color; + layout(location = 0) out vec4 fragColor; + layout(location = 1) out vec4 fragColor1; + void main() { + fragColor = texture(u_TexA, v_TexCoord) + texture(u_TexB, v_TexCoord) + imageLoad(u_Img, ivec2(0)) + texture(sampler2D(u_Tex, u_Smp), v_TexCoord); + fragColor1 = vec4(v_Color, 1.0); + } + """; + + private record Staged(byte[] spirv, ShaderManager.StageReflection reflection, ShaderManager.GraphicsBindingMap map) {} + + private static Staged stage(String source, int shaderKind, int glType) { + final SpirvCompiler.Result r = SpirvCompiler.compile(source, shaderKind, "codec", SpirvCompiler.Options.vulkanForced460Core()); + assertNotNull(r.spirv(), () -> "shaderc failed: " + r.error()); + final ByteBuffer spirv = r.spirv(); + try { + final ShaderManager.GraphicsBindingMap map = ShaderManager.remapSpirvForSDLGPU(spirv, glType); + final ShaderManager.StageReflection reflection = ShaderManager.reflectStage(spirv, glType == GL20.GL_VERTEX_SHADER); + return new Staged(ShaderCacheIO.toHeap(spirv), reflection, map); + } finally { + memFree(spirv); + } + } + + private static void assertReflectionEquals(ShaderManager.StageReflection e, ShaderManager.StageReflection a) { + assertNotNull(a); + assertEquals(e.counts(), a.counts()); + assertEquals(e.samplerNames(), a.samplerNames()); + assertEquals(e.extraUniformNames(), a.extraUniformNames()); + assertEquals(e.storageImageNames(), a.storageImageNames()); + assertEquals(e.uboSize(), a.uboSize()); + assertEquals(e.uboMembers(), a.uboMembers()); + assertEquals(e.vsInputs(), a.vsInputs()); + assertEquals(e.vsOutputs(), a.vsOutputs()); + assertEquals(e.fsInputs(), a.fsInputs()); + assertEquals(e.maxOutputLocation(), a.maxOutputLocation()); + assertEquals(e.numReadonlyStorageBuffers(), a.numReadonlyStorageBuffers()); + assertEquals(e.numReadwriteStorageBuffers(), a.numReadwriteStorageBuffers()); + assertEquals(e.numReadonlyStorageTextures(), a.numReadonlyStorageTextures()); + assertEquals(e.numReadwriteStorageTextures(), a.numReadwriteStorageTextures()); + assertEquals(e.blocks().length, a.blocks().length); + for (int b = 0; b < e.blocks().length; b++) assertEquals(e.blocks()[b], a.blocks()[b], "block " + b); + } + + private static void assertBindingMapEquals(ShaderManager.GraphicsBindingMap e, ShaderManager.GraphicsBindingMap a) { + assertNotNull(a); + assertArrayEquals(e.roStorageTextureGlSlots(), a.roStorageTextureGlSlots()); + assertArrayEquals(e.rwStorageTextureGlSlots(), a.rwStorageTextureGlSlots()); + assertArrayEquals(e.roSsboGlSlots(), a.roSsboGlSlots()); + } + + private static byte[] encodedVertex() { + final String src = ShaderTransformChain.inject(VERTEX, BLOCK, null); + final Staged s = stage(src, Shaderc.shaderc_vertex_shader, GL20.GL_VERTEX_SHADER); + return ShaderStageSerializer.encode(s.spirv(), s.reflection(), s.map(), Set.of("b"), src); + } + + @Test + void vertexStageRoundTrips() { + final String src = ShaderTransformChain.inject(VERTEX, BLOCK, null); + final Staged s = stage(src, Shaderc.shaderc_vertex_shader, GL20.GL_VERTEX_SHADER); + assertFalse(s.reflection().vsInputs().isEmpty()); + assertFalse(s.reflection().vsOutputs().isEmpty()); + assertTrue(s.reflection().uboMembers().stream().anyMatch(m -> m.arrayLen() > 1), "needs an array member"); + assertFalse(s.reflection().blocks()[ShaderManager.BLOCK_PER_FRAME].members().isEmpty(), "needs the per-frame block"); + + final byte[] enc = ShaderStageSerializer.encode(s.spirv(), s.reflection(), s.map(), Set.of("u_flag", "u_other"), src); + final ShaderStageSerializer.Stage d = ShaderStageSerializer.decode(enc); + assertNotNull(d); + assertArrayEquals(s.spirv(), d.spirv()); + assertReflectionEquals(s.reflection(), d.reflection()); + assertBindingMapEquals(s.map(), d.bindingMap()); + assertEquals(Set.of("u_flag", "u_other"), d.boolUniforms()); + assertEquals(src, d.source()); + } + + @Test + void fragmentStageRoundTrips() { + final Staged s = stage(FRAGMENT, Shaderc.shaderc_fragment_shader, GL20.GL_FRAGMENT_SHADER); + assertEquals(List.of("u_TexA", "u_TexB"), s.reflection().samplerNames()); + assertFalse(s.reflection().storageImageNames().isEmpty(), "needs a storage image"); + assertTrue(s.reflection().extraUniformNames().contains("u_Smp"), "needs a separate sampler"); + assertFalse(s.reflection().fsInputs().isEmpty()); + assertEquals(1, s.reflection().maxOutputLocation()); + + final ShaderStageSerializer.Stage d = ShaderStageSerializer.decode(ShaderStageSerializer.encode(s.spirv(), s.reflection(), s.map(), Set.of(), FRAGMENT)); + assertNotNull(d); + assertArrayEquals(s.spirv(), d.spirv()); + assertReflectionEquals(s.reflection(), d.reflection()); + assertBindingMapEquals(s.map(), d.bindingMap()); + assertTrue(d.boolUniforms().isEmpty()); + assertEquals(FRAGMENT, d.source()); + } + + @Test + void nullableFieldsSurvive() { + final byte[] spirv = {1, 2, 3, 4}; + final ShaderStageSerializer.Stage d = ShaderStageSerializer.decode(ShaderStageSerializer.encode(spirv, ShaderManager.StageReflection.EMPTY, ShaderManager.GraphicsBindingMap.EMPTY, null, null)); + assertNotNull(d); + assertArrayEquals(spirv, d.spirv()); + assertReflectionEquals(ShaderManager.StageReflection.EMPTY, d.reflection()); + assertBindingMapEquals(ShaderManager.GraphicsBindingMap.EMPTY, d.bindingMap()); + assertNull(d.source()); + assertTrue(d.boolUniforms().isEmpty()); + } + + @Test + void boolUniformOrderDoesNotChangeTheBytes() { + final byte[] spirv = {1, 2, 3, 4}; + final Set forward = new LinkedHashSet<>(List.of("a", "b", "c", "d")); + final Set backward = new LinkedHashSet<>(List.of("d", "c", "b", "a")); + final byte[] forwardBytes = ShaderStageSerializer.encode(spirv, ShaderManager.StageReflection.EMPTY, ShaderManager.GraphicsBindingMap.EMPTY, forward, "s"); + final byte[] backwardBytes = ShaderStageSerializer.encode(spirv, ShaderManager.StageReflection.EMPTY, ShaderManager.GraphicsBindingMap.EMPTY, backward, "s"); + assertArrayEquals(forwardBytes, backwardBytes); + } + + @Test + void transformRoundTripsAndSortsBools() { + final Set forward = new LinkedHashSet<>(List.of("a", "b", "c")); + final Set backward = new LinkedHashSet<>(List.of("c", "b", "a")); + final byte[] enc = ShaderStageSerializer.encodeTransform(FRAGMENT, forward); + assertArrayEquals(enc, ShaderStageSerializer.encodeTransform(FRAGMENT, backward)); + final ShaderManager.PrewarmTransformResult d = ShaderStageSerializer.decodeTransform(enc); + assertNotNull(d); + assertEquals(FRAGMENT, d.source()); + assertEquals(Set.of("a", "b", "c"), d.boolUniforms()); + assertTrue(ShaderStageSerializer.decodeTransform(ShaderStageSerializer.encodeTransform("", null)).boolUniforms().isEmpty()); + } + + @Test + void truncatedPayloadsAreRejected() { + final byte[] enc = encodedVertex(); + for (int len = 0; len < enc.length; len++) { + assertNull(ShaderStageSerializer.decode(Arrays.copyOf(enc, len)), "prefix of " + len + " bytes decoded"); + } + final byte[] transform = ShaderStageSerializer.encodeTransform(FRAGMENT, Set.of("a", "b")); + for (int len = 0; len < transform.length; len++) { + assertNull(ShaderStageSerializer.decodeTransform(Arrays.copyOf(transform, len)), "prefix of " + len + " bytes decoded"); + } + } + + @Test + void hugeLengthPrefixIsRejectedWithoutAllocating() { + final byte[] enc = encodedVertex(); + final byte[] bad = enc.clone(); + bad[0] = 0x7f; + bad[1] = (byte) 0xff; + bad[2] = (byte) 0xff; + bad[3] = (byte) 0xff; + assertNull(ShaderStageSerializer.decode(bad)); + } +} diff --git a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderTransformChain.java b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderTransformChain.java index cddc09d31..304460a31 100644 --- a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderTransformChain.java +++ b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/ShaderTransformChain.java @@ -1,8 +1,16 @@ package com.gtnewhorizons.angelica.sdlgpu.shader; +import com.gtnewhorizons.angelica.glsm.GlslTransformUtils; import com.gtnewhorizons.angelica.glsm.hooks.GLSMHooks; +import com.gtnewhorizons.angelica.glsm.hooks.PerFrameUniformBlock; import com.gtnewhorizons.angelica.glsm.shader.GlslVulkanPreprocess; +import com.gtnewhorizons.angelica.glsm.shader.GlslVulkanPreprocess.Edit; import org.lwjgl.opengl.GL20; +import org.taumc.glsl.grammar.GLSLParser; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.BiConsumer; final class ShaderTransformChain { @@ -12,12 +20,37 @@ final class ShaderTransformChain { final GlslVulkanPreprocess.Result pre = GlslVulkanPreprocess.run(source, glShaderType, "test", true); String src = pre != null ? pre.rewrittenSource() : source; if (glShaderType == GL20.GL_VERTEX_SHADER) { - src = ClipZRemap.injectGLToVulkanClipZ(src); + src = clipZ(src); } - src = SamplerStripper.stripUnused(src); + src = stripUnused(src); if (glShaderType == GL20.GL_VERTEX_SHADER || glShaderType == GL20.GL_FRAGMENT_SHADER) { - src = PerFrameBlockInjector.inject(src, GLSMHooks.perFrameUniformBlock, GLSMHooks.perPassUniformBlock); + src = inject(src, GLSMHooks.perFrameUniformBlock, GLSMHooks.perPassUniformBlock); } return src; } + + static String clipZ(String source) { + return edit(source, ClipZRemap::collectEdits); + } + + static String stripUnused(String source) { + if (!source.contains("sampler")) return source; + return edit(source, (root, edits) -> SamplerStripper.collectEdits(root, source, edits)); + } + + static String inject(String source, PerFrameUniformBlock perFrame, PerFrameUniformBlock perPass) { + return edit(source, (root, edits) -> PerFrameBlockInjector.collectEdits(root, perFrame, perPass, edits)); + } + + private static String edit(String source, BiConsumer> collect) { + final GLSLParser.Translation_unitContext root; + try { + root = GlslTransformUtils.parseFullQuiet(source); + } catch (Exception e) { + return source; + } + final List edits = new ArrayList<>(); + collect.accept(root, edits); + return edits.isEmpty() ? source : GlslVulkanPreprocess.applyEdits(source, edits); + } } diff --git a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/StageDiskCacheTest.java b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/StageDiskCacheTest.java new file mode 100644 index 000000000..3fc6b9eb5 --- /dev/null +++ b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/StageDiskCacheTest.java @@ -0,0 +1,324 @@ +package com.gtnewhorizons.angelica.sdlgpu.shader; + +import com.gtnewhorizons.angelica.glsm.hooks.GLSMHooks; +import com.gtnewhorizons.angelica.glsm.hooks.PerFrameUniformBlock; +import com.gtnewhorizons.angelica.glsm.hooks.PerFrameUniformBlock.Member; +import com.gtnewhorizons.angelica.glsm.shader.GlslVulkanPreprocess; +import com.gtnewhorizons.angelica.glsm.shader.ShaderCacheIO; +import com.gtnewhorizons.angelica.glsm.shader.ShaderDiskCache; +import com.gtnewhorizons.angelica.glsm.testutil.Reflect; +import com.gtnewhorizons.angelica.glsm.shader.UniformType; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.lwjgl.opengl.GL20; +import org.lwjgl.opengl.GL43; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class StageDiskCacheTest { + + private static final String RAW = """ + #version 460 core + in vec3 a_PosId; + void main() { + gl_Position = vec4(a_PosId, 1.0); + } + """; + + private static final String PFB_FRAG = """ + #version 460 core + uniform float frameTimeCounter; + layout(location = 0) out vec4 fragColor; + void main() { + fragColor = vec4(frameTimeCounter); + } + """; + + private static final String BROKEN = """ + #version 460 core + void main() { undeclared_var = 1.0; } + """; + + private static final byte[] POISON_SPIRV = {1, 2, 3, 4, 5, 6, 7, 8}; + + private static final PerFrameUniformBlock BLOCK_A = new PerFrameUniformBlock(List.of(new Member("frameTimeCounter", UniformType.FLOAT))); + + private static final PerFrameUniformBlock BLOCK_B = new PerFrameUniformBlock(List.of( + new Member("frameTimeCounter", UniformType.FLOAT), + new Member("rainStrength", UniformType.FLOAT))); + + @BeforeEach + @AfterEach + void cleanup() { + Reflect.setStatic(ShaderDiskCache.class, "root", null); + ShaderManager.clearPrewarmCache(); + GlslVulkanPreprocess.clearCache(); + GLSMHooks.perFrameUniformBlock = null; + GLSMHooks.perPassUniformBlock = null; + } + + private static byte[] poison(String marker) { + final ShaderManager.StageReflection reflection = new ShaderManager.StageReflection( + ShaderManager.ResourceCounts.EMPTY, List.of(marker), List.of(), List.of(), 0, List.of(), List.of(), List.of(), List.of(), + -1, 0, 0, 0, 0, ShaderManager.BlockReflection.emptyBlocks()); + return ShaderStageSerializer.encode(POISON_SPIRV, reflection, ShaderManager.GraphicsBindingMap.EMPTY, Set.of("poison_bool"), "poison_source"); + } + + private static ShaderManager.ShaderObject object(ShaderManager sm, int id) { + final Map objects = Reflect.get(sm, "shaderObjects"); + return objects.get(id); + } + + @Test + void prewarmHitReturnsTheStoredEntry(@TempDir Path dir) { + ShaderDiskCache.configure(dir, "t"); + ShaderManager.prewarmSpirv(RAW, GL20.GL_VERTEX_SHADER); + final ShaderDiskCache.Key key = ShaderManager.prewarmKey(RAW, GL20.GL_VERTEX_SHADER, null, null); + assertNotNull(ShaderDiskCache.get(key), "a successful prewarm must be stored"); + + ShaderManager.clearPrewarmCache(); + ShaderDiskCache.put(key, poison("poison_sampler")); + ShaderManager.prewarmSpirv(RAW, GL20.GL_VERTEX_SHADER); + + final ShaderManager sm = new ShaderManager(null); + final int shader = sm.createShader(GL20.GL_VERTEX_SHADER); + sm.shaderSource(shader, RAW); + assertEquals("poison_source", sm.getShaderSource(shader)); + sm.compileShader(shader); + + final ShaderManager.ShaderObject obj = object(sm, shader); + assertTrue(obj.compiled); + assertEquals(List.of("poison_sampler"), obj.reflection.samplerNames()); + assertEquals(Set.of("poison_bool"), obj.boolUniforms); + assertArrayEquals(POISON_SPIRV, ShaderCacheIO.toHeap(obj.spirv)); + sm.deleteShader(shader); + } + + @Test + void prewarmHitDoesNotShareTheSpirvBufferBetweenShaders(@TempDir Path dir) { + ShaderDiskCache.configure(dir, "t"); + ShaderManager.prewarmSpirv(RAW, GL20.GL_VERTEX_SHADER); + ShaderManager.clearPrewarmCache(); + ShaderManager.prewarmSpirv(RAW, GL20.GL_VERTEX_SHADER); + + final ShaderManager sm = new ShaderManager(null); + final int a = sm.createShader(GL20.GL_VERTEX_SHADER); + final int b = sm.createShader(GL20.GL_VERTEX_SHADER); + sm.shaderSource(a, RAW); + sm.shaderSource(b, RAW); + sm.compileShader(a); + sm.compileShader(b); + + final ShaderManager.ShaderObject objA = object(sm, a); + final ShaderManager.ShaderObject objB = object(sm, b); + assertTrue(objA.compiled && objB.compiled); + assertTrue(objA.spirv != objB.spirv); + assertArrayEquals(ShaderCacheIO.toHeap(objA.spirv), ShaderCacheIO.toHeap(objB.spirv)); + sm.deleteShader(a); + sm.deleteShader(b); + } + + @Test + void shaderSourceFallbackHitReturnsTheStoredEntry(@TempDir Path dir) { + ShaderDiskCache.configure(dir, "t"); + final ShaderManager first = new ShaderManager(null); + final int a = first.createShader(GL20.GL_VERTEX_SHADER); + first.shaderSource(a, RAW); + first.compileShader(a); + assertTrue(object(first, a).compiled); + final String transformed = first.getShaderSource(a); + final ShaderDiskCache.Key key = ShaderManager.stageKey(transformed, GL20.GL_VERTEX_SHADER); + assertNotNull(ShaderDiskCache.get(key), "a successful stage compile must be stored"); + assertFalse(Files.exists(dir.resolve("spirv")), "the stage entry already holds the SPIR-V"); + first.deleteShader(a); + + ShaderManager.clearPrewarmCache(); + ShaderDiskCache.put(key, poison("poison_sampler")); + + final ShaderManager sm = new ShaderManager(null); + final int b = sm.createShader(GL20.GL_VERTEX_SHADER); + sm.shaderSource(b, RAW); + assertEquals(transformed, sm.getShaderSource(b), "the transformed source must be available without compiling"); + sm.compileShader(b); + + final ShaderManager.ShaderObject obj = object(sm, b); + assertTrue(obj.compiled); + assertEquals(List.of("poison_sampler"), obj.reflection.samplerNames()); + assertArrayEquals(POISON_SPIRV, ShaderCacheIO.toHeap(obj.spirv)); + sm.deleteShader(b); + } + + @Test + void failedCompileStoresNothing(@TempDir Path dir) { + ShaderDiskCache.configure(dir, "t"); + ShaderManager.prewarmSpirv(BROKEN, GL20.GL_VERTEX_SHADER); + assertNull(ShaderDiskCache.get(ShaderManager.prewarmKey(BROKEN, GL20.GL_VERTEX_SHADER, null, null))); + + final ShaderManager sm = new ShaderManager(null); + final int shader = sm.createShader(GL20.GL_VERTEX_SHADER); + sm.shaderSource(shader, BROKEN); + sm.compileShader(shader); + assertFalse(object(sm, shader).compiled); + assertNull(ShaderDiskCache.get(ShaderManager.stageKey(sm.getShaderSource(shader), GL20.GL_VERTEX_SHADER))); + sm.deleteShader(shader); + } + + @Test + void corruptEntryFallsBackToCompilingAndIsReplaced(@TempDir Path dir) { + ShaderDiskCache.configure(dir, "t"); + final ShaderDiskCache.Key key = ShaderManager.prewarmKey(RAW, GL20.GL_VERTEX_SHADER, null, null); + ShaderDiskCache.put(key, new byte[] {9, 9, 9}); + ShaderManager.prewarmSpirv(RAW, GL20.GL_VERTEX_SHADER); + assertNotNull(ShaderStageSerializer.decode(ShaderDiskCache.get(key))); + } + + @Test + void computePrewarmRoundTripsThroughTheDisk(@TempDir Path dir) { + ShaderDiskCache.configure(dir, "t"); + final String compute = """ + #version 460 core + layout(local_size_x = 1) in; + layout(std430, binding = 0) buffer B { uint data[]; } b; + void main() { b.data[0] = 1u; } + """; + ShaderManager.prewarmSpirv(compute, GL43.GL_COMPUTE_SHADER); + final ShaderDiskCache.Key key = ShaderManager.prewarmKey(compute, GL43.GL_COMPUTE_SHADER, null, null); + assertNotNull(ShaderDiskCache.get(key)); + + ShaderManager.clearPrewarmCache(); + ShaderManager.prewarmSpirv(compute, GL43.GL_COMPUTE_SHADER); + final ShaderManager sm = new ShaderManager(null); + final int shader = sm.createShader(GL43.GL_COMPUTE_SHADER); + sm.shaderSource(shader, compute); + sm.compileShader(shader); + final ShaderManager.ShaderObject obj = object(sm, shader); + assertTrue(obj.compiled); + assertTrue(obj.reflection == ShaderManager.StageReflection.EMPTY, "a compute hit must keep the shared EMPTY reflection so link reflects the remapped copy"); + sm.deleteShader(shader); + } + + @Test + void prewarmEntryIsNotReusedAcrossPerFrameBlocks() { + GLSMHooks.perFrameUniformBlock = BLOCK_A; + ShaderManager.prewarmSpirv(PFB_FRAG, GL20.GL_FRAGMENT_SHADER); + + GLSMHooks.perFrameUniformBlock = BLOCK_B; + final ShaderManager sm = new ShaderManager(null); + final int viaTransform = sm.createShader(GL20.GL_FRAGMENT_SHADER); + sm.shaderSource(viaTransform, PFB_FRAG); + assertTrue(sm.getShaderSource(viaTransform).contains("rainStrength"), "the first prewarm's block must not leak into a different block"); + + ShaderManager.prewarmSpirv(PFB_FRAG, GL20.GL_FRAGMENT_SHADER); + final int viaPrewarm = sm.createShader(GL20.GL_FRAGMENT_SHADER); + sm.shaderSource(viaPrewarm, PFB_FRAG); + assertTrue(sm.getShaderSource(viaPrewarm).contains("rainStrength")); + + GLSMHooks.perFrameUniformBlock = BLOCK_A; + final int back = sm.createShader(GL20.GL_FRAGMENT_SHADER); + sm.shaderSource(back, PFB_FRAG); + assertFalse(sm.getShaderSource(back).contains("rainStrength")); + sm.shutdown(); + } + + private static byte[] poisonTransform() { + return ShaderStageSerializer.encodeTransform("poison_transform", Set.of("poison_bool")); + } + + @Test + void transformDiskHitReturnsTheStoredEntry(@TempDir Path dir) { + ShaderDiskCache.configure(dir, "t"); + final ShaderManager first = new ShaderManager(null); + final int a = first.createShader(GL20.GL_VERTEX_SHADER); + first.shaderSource(a, RAW); + final String transformed = first.getShaderSource(a); + final ShaderDiskCache.Key key = ShaderManager.transformDiskKey(RAW, GL20.GL_VERTEX_SHADER, null, null); + final ShaderManager.PrewarmTransformResult stored = ShaderStageSerializer.decodeTransform(ShaderDiskCache.get(key)); + assertNotNull(stored, "a successful transform must be stored"); + assertEquals(transformed, stored.source()); + first.compileShader(a); + first.deleteShader(a); + + ShaderManager.clearPrewarmCache(); + ShaderDiskCache.put(key, poisonTransform()); + + final ShaderManager sm = new ShaderManager(null); + final int b = sm.createShader(GL20.GL_VERTEX_SHADER); + sm.shaderSource(b, RAW); + assertEquals("poison_transform", sm.getShaderSource(b)); + assertEquals(Set.of("poison_bool"), object(sm, b).boolUniforms); + sm.compileShader(b); + sm.deleteShader(b); + } + + @Test + void corruptTransformEntryFallsBackToTransformingAndIsReplaced(@TempDir Path dir) { + ShaderDiskCache.configure(dir, "t"); + final ShaderDiskCache.Key key = ShaderManager.transformDiskKey(RAW, GL20.GL_VERTEX_SHADER, null, null); + ShaderDiskCache.put(key, new byte[] {9, 9, 9}); + + final ShaderManager sm = new ShaderManager(null); + final int shader = sm.createShader(GL20.GL_VERTEX_SHADER); + sm.shaderSource(shader, RAW); + assertEquals(ShaderManager.applyPrewarmTransformsFull(RAW, GL20.GL_VERTEX_SHADER, null, null).source(), sm.getShaderSource(shader)); + assertNotNull(ShaderStageSerializer.decodeTransform(ShaderDiskCache.get(key))); + sm.compileShader(shader); + sm.deleteShader(shader); + } + + @Test + void transformKeyChangesWithEveryInput() { + final String base = ShaderManager.transformDiskKey(RAW, GL20.GL_VERTEX_SHADER, BLOCK_A, null).hex(); + + assertEquals(base, ShaderManager.transformDiskKey(RAW, GL20.GL_VERTEX_SHADER, new PerFrameUniformBlock(List.of(new Member("frameTimeCounter", UniformType.FLOAT))), null).hex()); + assertNotEquals(base, ShaderManager.transformDiskKey(RAW + " ", GL20.GL_VERTEX_SHADER, BLOCK_A, null).hex(), "source"); + assertNotEquals(base, ShaderManager.transformDiskKey(RAW, GL20.GL_FRAGMENT_SHADER, BLOCK_A, null).hex(), "type"); + assertNotEquals(base, ShaderManager.transformDiskKey(RAW, GL20.GL_VERTEX_SHADER, BLOCK_B, null).hex(), "per-frame block"); + assertNotEquals(base, ShaderManager.transformDiskKey(RAW, GL20.GL_VERTEX_SHADER, null, null).hex(), "no per-frame block"); + assertNotEquals(base, ShaderManager.transformDiskKey(RAW, GL20.GL_VERTEX_SHADER, BLOCK_A, BLOCK_B).hex(), "per-pass block"); + assertNotEquals(base, ShaderManager.transformDiskKey(RAW, GL20.GL_VERTEX_SHADER, null, BLOCK_A).hex(), "slots are not interchangeable"); + assertNotEquals(base, ShaderManager.prewarmKey(RAW, GL20.GL_VERTEX_SHADER, null, null).hex(), "layer and toolchain ids"); + } + + @Test + void stageKeyIgnoresUniformBlocks() { + GLSMHooks.perFrameUniformBlock = BLOCK_A; + final String base = ShaderManager.stageKey(RAW, GL20.GL_VERTEX_SHADER).hex(); + GLSMHooks.perFrameUniformBlock = BLOCK_B; + GLSMHooks.perPassUniformBlock = BLOCK_A; + assertEquals(base, ShaderManager.stageKey(RAW, GL20.GL_VERTEX_SHADER).hex()); + assertNotEquals(base, ShaderManager.stageKey(RAW + " ", GL20.GL_VERTEX_SHADER).hex()); + assertNotEquals(base, ShaderManager.stageKey(RAW, GL20.GL_FRAGMENT_SHADER).hex()); + } + + @Test + void prewarmKeyChangesWithEveryInput() { + final String base = ShaderManager.prewarmKey(RAW, GL20.GL_VERTEX_SHADER, BLOCK_A, null).hex(); + + assertEquals(base, ShaderManager.prewarmKey(RAW, GL20.GL_VERTEX_SHADER, BLOCK_A, null).hex()); + assertEquals(base, ShaderManager.prewarmKey(RAW, GL20.GL_VERTEX_SHADER, new PerFrameUniformBlock(List.of(new Member("frameTimeCounter", UniformType.FLOAT))), null).hex(), "equal blocks built separately must agree"); + + assertNotEquals(base, ShaderManager.stageKey(RAW, GL20.GL_VERTEX_SHADER).hex()); + assertNotEquals(base, ShaderManager.prewarmKey(RAW + " ", GL20.GL_VERTEX_SHADER, BLOCK_A, null).hex()); + assertNotEquals(base, ShaderManager.prewarmKey(RAW, GL20.GL_FRAGMENT_SHADER, BLOCK_A, null).hex()); + + assertNotEquals(base, ShaderManager.prewarmKey(RAW, GL20.GL_VERTEX_SHADER, BLOCK_B, null).hex(), "an added per-frame member"); + assertNotEquals(base, ShaderManager.prewarmKey(RAW, GL20.GL_VERTEX_SHADER, new PerFrameUniformBlock(List.of(new Member("frameTimeCounter", UniformType.INT))), null).hex(), "a changed member type"); + assertNotEquals(base, ShaderManager.prewarmKey(RAW, GL20.GL_VERTEX_SHADER, new PerFrameUniformBlock(List.of(new Member("rainStrength", UniformType.FLOAT))), null).hex(), "a renamed member"); + assertNotEquals(base, ShaderManager.prewarmKey(RAW, GL20.GL_VERTEX_SHADER, null, null).hex(), "no per-frame block"); + assertNotEquals(base, ShaderManager.prewarmKey(RAW, GL20.GL_VERTEX_SHADER, BLOCK_A, BLOCK_B).hex(), "a per-pass block"); + } +} diff --git a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/dxbc/DxbcCompileTest.java b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/dxbc/DxbcCompileTest.java index a8f589d6e..6e2309a0f 100644 --- a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/dxbc/DxbcCompileTest.java +++ b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/dxbc/DxbcCompileTest.java @@ -1,17 +1,26 @@ package com.gtnewhorizons.angelica.sdlgpu.shader.dxbc; +import com.gtnewhorizons.angelica.glsm.shader.ShaderCacheIO; +import com.gtnewhorizons.angelica.glsm.shader.ShaderDiskCache; import com.gtnewhorizons.angelica.glsm.shader.SpirvCompiler; +import com.gtnewhorizons.angelica.glsm.testutil.Reflect; import com.gtnewhorizons.angelica.sdlgpu.shader.ShaderManager; import com.gtnewhorizons.angelica.sdlgpu.shader.SpirvTestShaders; +import com.gtnewhorizons.angelica.sdlgpu.shader.cross.CrossCompileCache; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledOnOs; import org.junit.jupiter.api.condition.OS; +import org.junit.jupiter.api.io.TempDir; import org.lwjgl.opengl.GL20; import org.lwjgl.opengl.GL43; import org.lwjgl.system.MemoryUtil; import org.lwjgl.util.shaderc.Shaderc; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -21,6 +30,13 @@ import static org.junit.jupiter.api.Assertions.fail; @EnabledOnOs(OS.WINDOWS) class DxbcCompileTest { + private static CrossCompileCache cache() { return Reflect.getStatic(DxbcCrossCompile.class, "CACHE"); } + + @AfterEach + void resetDiskCache() { + Reflect.setStatic(ShaderDiskCache.class, "root", null); + Reflect.>get(cache(), "cache").clear(); + } private static final byte DXBC_MAGIC_D = 'D'; private static final byte DXBC_MAGIC_X = 'X'; @@ -34,7 +50,7 @@ class DxbcCompileTest { final ByteBuffer spirv = compile(SpirvTestShaders.VERTEX_GLSL, Shaderc.shaderc_vertex_shader); try { ShaderManager.remapSpirvForSDLGPU(spirv, GL20.GL_VERTEX_SHADER); - final DxbcCrossCompile.Output out = DxbcCrossCompile.compile(spirv, GL20.GL_VERTEX_SHADER); + final CrossCompileCache.Output out = DxbcCrossCompile.compile(spirv, GL20.GL_VERTEX_SHADER); try { assertNotNull(out.code()); assertTrue(out.code().remaining() >= 4, "DXBC output too short"); @@ -53,7 +69,7 @@ class DxbcCompileTest { final ByteBuffer spirv = compile(SpirvTestShaders.FRAGMENT_GLSL, Shaderc.shaderc_fragment_shader); try { ShaderManager.remapSpirvForSDLGPU(spirv, GL20.GL_FRAGMENT_SHADER); - final DxbcCrossCompile.Output out = DxbcCrossCompile.compile(spirv, GL20.GL_FRAGMENT_SHADER); + final CrossCompileCache.Output out = DxbcCrossCompile.compile(spirv, GL20.GL_FRAGMENT_SHADER); try { assertNotNull(out.code()); assertTrue(out.code().remaining() >= 4, "DXBC output too short"); @@ -73,7 +89,7 @@ class DxbcCompileTest { final ByteBuffer spirv = compile(SpirvTestShaders.HIZ_INIT_GLSL, Shaderc.shaderc_compute_shader); try { ShaderManager.remapSpirvForComputeSDLGPU(spirv); - final DxbcCrossCompile.Output out = DxbcCrossCompile.compile(spirv, GL43.GL_COMPUTE_SHADER); + final CrossCompileCache.Output out = DxbcCrossCompile.compile(spirv, GL43.GL_COMPUTE_SHADER); try { assertNotNull(out.code()); assertTrue(out.code().remaining() >= 4, "DXBC compute output too short"); @@ -105,7 +121,7 @@ class DxbcCompileTest { final ByteBuffer spirv = compile(SpirvTestShaders.VERTEX_GLSL, Shaderc.shaderc_vertex_shader); try { ShaderManager.remapSpirvForSDLGPU(spirv, GL20.GL_VERTEX_SHADER); - final DxbcCrossCompile.Output out = DxbcCrossCompile.compile(spirv, GL20.GL_VERTEX_SHADER); + final CrossCompileCache.Output out = DxbcCrossCompile.compile(spirv, GL20.GL_VERTEX_SHADER); MemoryUtil.memFree(out.code()); } finally { MemoryUtil.memFree(spirv); @@ -113,6 +129,29 @@ class DxbcCompileTest { } } + @Test + void diskCacheHitReturnsStoredBlob(@TempDir Path dir) { + ShaderDiskCache.configure(dir, "t"); + Reflect.>get(cache(), "cache").clear(); + final ByteBuffer spirv = compile(SpirvTestShaders.VERTEX_GLSL, Shaderc.shaderc_vertex_shader); + try { + ShaderManager.remapSpirvForSDLGPU(spirv, GL20.GL_VERTEX_SHADER); + final byte[] heap = ShaderCacheIO.toHeap(spirv); + MemoryUtil.memFree(DxbcCrossCompile.compile(spirv, GL20.GL_VERTEX_SHADER).code()); + Reflect.>get(cache(), "cache").clear(); + ShaderDiskCache.putBlob(cache().diskKey(heap, GL20.GL_VERTEX_SHADER), "poison_entry", "poison".getBytes(StandardCharsets.UTF_8)); + final CrossCompileCache.Output out = DxbcCrossCompile.compile(spirv, GL20.GL_VERTEX_SHADER); + try { + assertEquals("poison_entry", out.entrypoint()); + assertEquals("poison", new String(ShaderCacheIO.toHeap(out.code()), StandardCharsets.UTF_8)); + } finally { + MemoryUtil.memFree(out.code()); + } + } finally { + MemoryUtil.memFree(spirv); + } + } + private static void assertDxbcMagic(ByteBuffer dxbc) { final ByteBuffer view = dxbc.duplicate(); assertEquals(DXBC_MAGIC_D, view.get(0), "DXBC magic byte 0"); diff --git a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/msl/FragmentStorageTextureSlotTest.java b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/msl/FragmentStorageTextureSlotTest.java index a9f3597c1..a409019bc 100644 --- a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/msl/FragmentStorageTextureSlotTest.java +++ b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/msl/FragmentStorageTextureSlotTest.java @@ -4,6 +4,7 @@ import com.gtnewhorizons.angelica.glsm.shader.GlslVulkanPreprocess; import com.gtnewhorizons.angelica.glsm.shader.SpirvCompiler; import com.gtnewhorizons.angelica.glsm.testutil.Reflect; import com.gtnewhorizons.angelica.sdlgpu.shader.ShaderManager; +import com.gtnewhorizons.angelica.sdlgpu.shader.cross.CrossCompileCache; import org.junit.jupiter.api.Test; import org.lwjgl.opengl.GL20; import org.lwjgl.system.MemoryUtil; @@ -56,7 +57,7 @@ class FragmentStorageTextureSlotTest { final ByteBuffer spirv = r.spirv(); try { ShaderManager.remapSpirvForSDLGPU(spirv, GL20.GL_FRAGMENT_SHADER); - final MslCrossCompile.Output out = MslCrossCompile.compile(spirv, GL20.GL_FRAGMENT_SHADER); + final CrossCompileCache.Output out = MslCrossCompile.compile(spirv, GL20.GL_FRAGMENT_SHADER); try { final String msl = StandardCharsets.UTF_8.decode(out.code().duplicate()).toString(); final boolean[] seen = new boolean[samplerCount]; @@ -135,7 +136,7 @@ class FragmentStorageTextureSlotTest { final ByteBuffer spirv = Reflect.get(obj, "spirv"); final ShaderManager.StageReflection refl = Reflect.get(obj, "reflection"); - final MslCrossCompile.Output out = MslCrossCompile.compile(spirv, GL20.GL_FRAGMENT_SHADER); + final CrossCompileCache.Output out = MslCrossCompile.compile(spirv, GL20.GL_FRAGMENT_SHADER); try { final String msl = StandardCharsets.UTF_8.decode(out.code().duplicate()).toString(); final int samplerCount = refl.counts().numSamplers(); diff --git a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/msl/MslCrossCompileTest.java b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/msl/MslCrossCompileTest.java index 3929e2b5e..7f869facb 100644 --- a/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/msl/MslCrossCompileTest.java +++ b/sdl-gpu/src/test/java/com/gtnewhorizons/angelica/sdlgpu/shader/msl/MslCrossCompileTest.java @@ -1,9 +1,15 @@ package com.gtnewhorizons.angelica.sdlgpu.shader.msl; +import com.gtnewhorizons.angelica.glsm.shader.ShaderCacheIO; +import com.gtnewhorizons.angelica.glsm.shader.ShaderDiskCache; import com.gtnewhorizons.angelica.glsm.shader.SpirvCompiler; +import com.gtnewhorizons.angelica.glsm.testutil.Reflect; import com.gtnewhorizons.angelica.sdlgpu.shader.ShaderManager; import com.gtnewhorizons.angelica.sdlgpu.shader.SpirvTestShaders; +import com.gtnewhorizons.angelica.sdlgpu.shader.cross.CrossCompileCache; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.lwjgl.opengl.GL20; import org.lwjgl.opengl.GL43; import org.lwjgl.system.MemoryUtil; @@ -15,6 +21,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashSet; +import java.util.Map; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -28,6 +35,13 @@ import static org.junit.jupiter.api.Assertions.fail; * Tests SPIR-V → MSL cross-compilation for the macOS/Metal backend. */ class MslCrossCompileTest { + private static CrossCompileCache cache() { return Reflect.getStatic(MslCrossCompile.class, "CACHE"); } + + @AfterEach + void resetDiskCache() { + Reflect.setStatic(ShaderDiskCache.class, "root", null); + Reflect.>get(cache(), "cache").clear(); + } @@ -36,7 +50,7 @@ class MslCrossCompileTest { final ByteBuffer spirv = compile(SpirvTestShaders.VERTEX_GLSL, Shaderc.shaderc_vertex_shader); try { ShaderManager.remapSpirvForSDLGPU(spirv, GL20.GL_VERTEX_SHADER); - final MslCrossCompile.Output out = MslCrossCompile.compile(spirv, GL20.GL_VERTEX_SHADER); + final CrossCompileCache.Output out = MslCrossCompile.compile(spirv, GL20.GL_VERTEX_SHADER); try { final String msl = decode(out.code()); assertTrue(msl.length() > 0, "MSL output non-empty"); @@ -59,7 +73,7 @@ class MslCrossCompileTest { final ByteBuffer spirv = compile(SpirvTestShaders.FRAGMENT_GLSL, Shaderc.shaderc_fragment_shader); try { ShaderManager.remapSpirvForSDLGPU(spirv, GL20.GL_FRAGMENT_SHADER); - final MslCrossCompile.Output out = MslCrossCompile.compile(spirv, GL20.GL_FRAGMENT_SHADER); + final CrossCompileCache.Output out = MslCrossCompile.compile(spirv, GL20.GL_FRAGMENT_SHADER); try { final String msl = decode(out.code()); assertTrue(msl.contains("fragment "), "MSL should contain fragment stage qualifier:\n" + msl); @@ -98,7 +112,7 @@ class MslCrossCompileTest { final ByteBuffer spirv = compile(SpirvTestShaders.LDS_MULTI_MIP_GLSL, Shaderc.shaderc_compute_shader); try { ShaderManager.remapSpirvForComputeSDLGPU(spirv); - final MslCrossCompile.Output out = MslCrossCompile.compile(spirv, GL43.GL_COMPUTE_SHADER); + final CrossCompileCache.Output out = MslCrossCompile.compile(spirv, GL43.GL_COMPUTE_SHADER); try { final String msl = decode(out.code()); assertTrue(msl.contains("kernel "), "Compute MSL should contain kernel stage qualifier:\n" + msl); @@ -130,7 +144,7 @@ class MslCrossCompileTest { final ByteBuffer spirv = compile(glsl, Shaderc.shaderc_compute_shader); try { ShaderManager.remapSpirvForComputeSDLGPU(spirv); - final MslCrossCompile.Output out = MslCrossCompile.compile(spirv, GL43.GL_COMPUTE_SHADER); + final CrossCompileCache.Output out = MslCrossCompile.compile(spirv, GL43.GL_COMPUTE_SHADER); try { final String msl = decode(out.code()); assertTrue(msl.contains("kernel "), "compute MSL should contain kernel stage qualifier:\n" + msl); @@ -151,7 +165,7 @@ class MslCrossCompileTest { final ByteBuffer spirv = compile(CHUNK_CULL_GLSL, Shaderc.shaderc_compute_shader); try { ShaderManager.remapSpirvForComputeSDLGPU(spirv); - final MslCrossCompile.Output out = MslCrossCompile.compile(spirv, GL43.GL_COMPUTE_SHADER); + final CrossCompileCache.Output out = MslCrossCompile.compile(spirv, GL43.GL_COMPUTE_SHADER); try { final String msl = decode(out.code()); assertTrue(msl.contains("kernel "), "Compute MSL should contain kernel stage qualifier:\n" + msl); @@ -186,7 +200,7 @@ class MslCrossCompileTest { final ByteBuffer spirv = compile(VERTEX_WRITES_IMAGE_GLSL, Shaderc.shaderc_vertex_shader); try { ShaderManager.remapSpirvForSDLGPU(spirv, GL20.GL_VERTEX_SHADER); - final MslCrossCompile.Output out = MslCrossCompile.compile(spirv, GL20.GL_VERTEX_SHADER); + final CrossCompileCache.Output out = MslCrossCompile.compile(spirv, GL20.GL_VERTEX_SHADER); try { final String msl = decode(out.code()); assertFalse(msl.contains("vertex void main0"), @@ -207,7 +221,7 @@ class MslCrossCompileTest { final ByteBuffer spirv = compile(SpirvTestShaders.VERTEX_GLSL, Shaderc.shaderc_vertex_shader); try { ShaderManager.remapSpirvForSDLGPU(spirv, GL20.GL_VERTEX_SHADER); - final MslCrossCompile.Output out = MslCrossCompile.compile(spirv, GL20.GL_VERTEX_SHADER); + final CrossCompileCache.Output out = MslCrossCompile.compile(spirv, GL20.GL_VERTEX_SHADER); MemoryUtil.memFree(out.code()); } finally { MemoryUtil.memFree(spirv); @@ -333,7 +347,7 @@ class MslCrossCompileTest { final ByteBuffer spirv = compile(glsl, Shaderc.shaderc_compute_shader); try { ShaderManager.remapSpirvForComputeSDLGPU(spirv); - final MslCrossCompile.Output out = MslCrossCompile.compile(spirv, GL43.GL_COMPUTE_SHADER); + final CrossCompileCache.Output out = MslCrossCompile.compile(spirv, GL43.GL_COMPUTE_SHADER); try { final String msl = decode(out.code()); assertUniqueBufferSlots(msl); @@ -388,7 +402,7 @@ class MslCrossCompileTest { final ByteBuffer spirv = compile(glsl, Shaderc.shaderc_compute_shader); try { ShaderManager.remapSpirvForComputeSDLGPU(spirv); - final MslCrossCompile.Output out = MslCrossCompile.compile(spirv, GL43.GL_COMPUTE_SHADER); + final CrossCompileCache.Output out = MslCrossCompile.compile(spirv, GL43.GL_COMPUTE_SHADER); try { assertTrue(decode(out.code()).contains("kernel "), name + " should produce compute MSL"); } finally { @@ -400,6 +414,29 @@ class MslCrossCompileTest { } } + @Test + void diskCacheHitReturnsStoredBlob(@TempDir Path dir) { + ShaderDiskCache.configure(dir, "t"); + Reflect.>get(cache(), "cache").clear(); + final ByteBuffer spirv = compile(SpirvTestShaders.VERTEX_GLSL, Shaderc.shaderc_vertex_shader); + try { + ShaderManager.remapSpirvForSDLGPU(spirv, GL20.GL_VERTEX_SHADER); + final byte[] heap = ShaderCacheIO.toHeap(spirv); + MemoryUtil.memFree(MslCrossCompile.compile(spirv, GL20.GL_VERTEX_SHADER).code()); + Reflect.>get(cache(), "cache").clear(); + ShaderDiskCache.putBlob(cache().diskKey(heap, GL20.GL_VERTEX_SHADER), "poison_entry", "poison".getBytes(StandardCharsets.UTF_8)); + final CrossCompileCache.Output out = MslCrossCompile.compile(spirv, GL20.GL_VERTEX_SHADER); + try { + assertEquals("poison_entry", out.entrypoint()); + assertEquals("poison", new String(ShaderCacheIO.toHeap(out.code()), StandardCharsets.UTF_8)); + } finally { + MemoryUtil.memFree(out.code()); + } + } finally { + MemoryUtil.memFree(spirv); + } + } + private static String decode(ByteBuffer buf) { final ByteBuffer slice = buf.duplicate(); int len = slice.remaining(); diff --git a/src/main/java/com/gtnewhorizons/angelica/proxy/ClientProxy.java b/src/main/java/com/gtnewhorizons/angelica/proxy/ClientProxy.java index 8a2b55057..2c31cb733 100644 --- a/src/main/java/com/gtnewhorizons/angelica/proxy/ClientProxy.java +++ b/src/main/java/com/gtnewhorizons/angelica/proxy/ClientProxy.java @@ -4,6 +4,7 @@ import com.gtnewhorizons.angelica.debug.DebugKeyHandler; import com.gtnewhorizons.angelica.rendering.culling.GpuCulling; import static com.gtnewhorizons.angelica.AngelicaMod.MOD_ID; +import java.io.File; import java.io.IOException; import java.lang.management.ManagementFactory; import java.util.Locale; @@ -40,6 +41,7 @@ import com.gtnewhorizon.gtnhlib.client.renderer.vao.VAOManager; import com.gtnewhorizons.angelica.client.font.DarkModeUtils; import com.gtnewhorizons.angelica.commands.AngelicaCommand; import com.gtnewhorizons.angelica.AngelicaMod; +import com.gtnewhorizons.angelica.Tags; import com.gtnewhorizons.angelica.common.BlockError; import com.gtnewhorizons.angelica.common.BlockIsbrhTest; import com.gtnewhorizons.angelica.compat.ModStatus; @@ -62,6 +64,7 @@ import com.gtnewhorizons.angelica.dynamiclights.config.EntityLightConfig; import com.gtnewhorizons.angelica.glsm.GLStateManager; import com.gtnewhorizons.angelica.glsm.backend.VSyncMode; import com.gtnewhorizons.angelica.glsm.profiling.Tracy; +import com.gtnewhorizons.angelica.glsm.shader.ShaderDiskCache; import com.gtnewhorizons.angelica.hudcaching.HUDCaching; import com.gtnewhorizons.angelica.iris.IrisGLSMBridge; import com.gtnewhorizons.angelica.mixins.interfaces.IGameSettingsExt; @@ -148,6 +151,11 @@ public final class ClientProxy extends CommonProxy { BlockIsbrhTest.renderId = RenderingRegistry.getNextAvailableRenderId(); RenderingRegistry.registerBlockHandler(BlockIsbrhTest.renderId, new IsbrhTestRenderer()); } + final File source = event.getSourceFile(); + if (source != null && source.isFile()) { + ShaderDiskCache.configure(Minecraft.getMinecraft().mcDataDir.toPath().resolve("angelica").resolve("shadercache"), + Tags.VERSION + "|" + source.length() + "|" + source.lastModified()); + } if (AngelicaConfig.enableIris) { IrisGLSMBridge.installImmediateExtendedHandler(); Iris.warmupShaderTransforms(); diff --git a/src/main/java/net/coderbot/iris/pipeline/transform/RwImageStoreExtractor.java b/src/main/java/net/coderbot/iris/pipeline/transform/RwImageStoreExtractor.java index e3e89c97b..e8e58d698 100644 --- a/src/main/java/net/coderbot/iris/pipeline/transform/RwImageStoreExtractor.java +++ b/src/main/java/net/coderbot/iris/pipeline/transform/RwImageStoreExtractor.java @@ -58,6 +58,10 @@ public final class RwImageStoreExtractor { private static volatile Map activeCustomImages = Map.of(); + static Map activeCustomImages() { + return activeCustomImages; + } + public static void setActiveCustomImages(Map images) { activeCustomImages = (images == null) ? Map.of() : Map.copyOf(images); } diff --git a/src/main/java/net/coderbot/iris/pipeline/transform/ShaderTransformer.java b/src/main/java/net/coderbot/iris/pipeline/transform/ShaderTransformer.java index f84576e9e..403db8188 100644 --- a/src/main/java/net/coderbot/iris/pipeline/transform/ShaderTransformer.java +++ b/src/main/java/net/coderbot/iris/pipeline/transform/ShaderTransformer.java @@ -8,6 +8,7 @@ import com.gtnewhorizons.angelica.glsm.backend.BackendManager; import com.gtnewhorizons.angelica.rendering.celeritas.iris.IrisExtendedChunkVertexType; import it.unimi.dsi.fastutil.objects.Object2ObjectLinkedOpenHashMap; import net.coderbot.iris.Iris; +import com.gtnewhorizons.angelica.glsm.shader.ShaderDiskCache; import com.gtnewhorizons.angelica.glsm.shader.ShaderType; import net.coderbot.iris.pipeline.transform.parameter.AttributeParameters; import net.coderbot.iris.pipeline.transform.parameter.Parameters; @@ -22,13 +23,19 @@ import org.taumc.glsl.Transformer; import org.taumc.glsl.grammar.GLSLLexer; import org.taumc.glsl.grammar.GLSLParser; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.EnumMap; import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.BooleanSupplier; +import java.util.function.Function; +import java.util.function.Supplier; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -226,6 +233,57 @@ public class ShaderTransformer { record StageArtifact(GLSLParser.Translation_unitContext tree, int headerLen) {} + static ShaderDiskCache.Key diskKey(Patch patch, EnumMap inputs, Parameters parameters) { + final ShaderDiskCache.Key k = ShaderDiskCache.key("iris-transform").str(patch.name()); + parameters.appendDiskKey(k); + k.i(RenderSystem.getMaxGlslVersion()).b(RenderSystem.supportsSSBO()).b(RenderSystem.supportsImageLoadStore()) + .b(RenderSystem.isGLES()).b(BackendManager.RENDER_BACKEND.isSDLGPU()).i(maxSupportedHoistVersion); + if (enabledRequirements == null) { + k.i(-1); + } else { + k.i(enabledRequirements.length); + for (VersionRequirement r : enabledRequirements) k.str(r.keyword()); + } + appendSorted(k, SamplerToStorageImageRewriter.activeCandidates(), Object::toString); + appendSorted(k, RwImageStoreExtractor.activeCustomImages().entrySet(), e -> e.getKey() + "=" + e.getValue()); + if (patch == Patch.CELERITAS_TERRAIN) k.str(CeleritasHeader.VALUE); + for (PatchShaderType t : PatchShaderType.VALUES) k.str(inputs.get(t)); + return k; + } + + private static void appendSorted(ShaderDiskCache.Key k, Collection items, Function render) { + final List rendered = new ArrayList<>(items.size()); + for (T item : items) rendered.add(render.apply(item)); + k.sortedStrs(rendered); + } + + private static Map diskOrCompute(Patch patchType, EnumMap inputs, Parameters parameters, Supplier> compute) { + final ShaderDiskCache.Key diskKey = ShaderDiskCache.isEnabled() ? diskKey(patchType, inputs, parameters) : null; + final Map fromDisk = diskKey != null ? readDisk(diskKey) : null; + if (fromDisk != null) return fromDisk; + final Map result = compute.get(); + if (diskKey != null) writeDisk(diskKey, result); + return result; + } + + private static Map readDisk(ShaderDiskCache.Key key) { + final Map stored = ShaderDiskCache.getStrings(key); + if (stored == null) return null; + try { + final EnumMap out = new EnumMap<>(PatchShaderType.class); + for (Map.Entry e : stored.entrySet()) out.put(PatchShaderType.valueOf(e.getKey()), e.getValue()); + return out; + } catch (RuntimeException e) { + return null; + } + } + + private static void writeDisk(ShaderDiskCache.Key key, Map result) { + final Map out = new LinkedHashMap<>(); + for (Map.Entry e : result.entrySet()) out.put(e.getKey().name(), e.getValue()); + ShaderDiskCache.putStrings(key, out); + } + public static

Map transform(String vertex, String geometry, String tessControl, String tessEval, String fragment, P parameters) { return transform(vertex, geometry, tessControl, tessEval, fragment, parameters, null); } @@ -251,7 +309,7 @@ public class ShaderTransformer { result = shaderTransformationCache.getAndMoveToLast(key); } if(result == null || !useCache) { - result = transformInternal(inputs, patchType, parameters, artifactsOut); + result = diskOrCompute(patchType, inputs, parameters, () -> transformInternal(inputs, patchType, parameters, artifactsOut)); // Clear this, we don't want whatever random type was last transformed being considered for the key parameters.type = null; synchronized (shaderTransformationCache) { @@ -293,7 +351,7 @@ public class ShaderTransformer { result = shaderTransformationCache.getAndMoveToLast(key); } if (result == null || !useCache) { - result = transformComputeInternal(compute, patchType, parameters, artifactsOut); + result = diskOrCompute(patchType, inputs, parameters, () -> transformComputeInternal(compute, patchType, parameters, artifactsOut)); // Clear this, we don't want whatever random type was last transformed being considered for the key parameters.type = null; synchronized (shaderTransformationCache) { @@ -481,7 +539,7 @@ public class ShaderTransformer { String headerTail = stageHeader.extensions(); // For Celeritas terrain vertex shaders, inject chunk_vertex.glsl header if (patchType == Patch.CELERITAS_TERRAIN && shaderType == PatchShaderType.VERTEX) { - headerTail += computeCeleritasHeader(); + headerTail += CeleritasHeader.VALUE; } final StringBuilder formattedShaderBuilder = new StringBuilder(); @@ -664,6 +722,10 @@ public class ShaderTransformer { } } + private static final class CeleritasHeader { + static final String VALUE = computeCeleritasHeader(); + } + private static String computeCeleritasHeader() { final ShaderConstants constants = ShaderConstants.builder() .add("VERT_POS_SCALE", "1.0") diff --git a/src/main/java/net/coderbot/iris/pipeline/transform/parameter/AttributeParameters.java b/src/main/java/net/coderbot/iris/pipeline/transform/parameter/AttributeParameters.java index 5ae56d32c..8c729589e 100644 --- a/src/main/java/net/coderbot/iris/pipeline/transform/parameter/AttributeParameters.java +++ b/src/main/java/net/coderbot/iris/pipeline/transform/parameter/AttributeParameters.java @@ -1,6 +1,7 @@ package net.coderbot.iris.pipeline.transform.parameter; import com.gtnewhorizons.angelica.glsm.ffp.Instancing; +import com.gtnewhorizons.angelica.glsm.shader.ShaderDiskCache; import net.coderbot.iris.gbuffer_overrides.matching.InputAvailability; import net.coderbot.iris.pipeline.transform.Patch; import net.coderbot.iris.shaderpack.texture.TextureStage; @@ -27,6 +28,12 @@ public class AttributeParameters extends Parameters { return TextureStage.GBUFFERS_AND_SHADOW; } + @Override + public void appendDiskKey(ShaderDiskCache.Key k) { + super.appendDiskKey(k); + k.b(hasGeometry).b(hasTesselation).str(inputs == null ? null : inputs.name()).b(scrollGlint).str(instancing.name()); + } + @Override public int hashCode() { final int prime = 31; diff --git a/src/main/java/net/coderbot/iris/pipeline/transform/parameter/Parameters.java b/src/main/java/net/coderbot/iris/pipeline/transform/parameter/Parameters.java index 07d8e57da..7bfec3b57 100644 --- a/src/main/java/net/coderbot/iris/pipeline/transform/parameter/Parameters.java +++ b/src/main/java/net/coderbot/iris/pipeline/transform/parameter/Parameters.java @@ -2,12 +2,16 @@ package net.coderbot.iris.pipeline.transform.parameter; import it.unimi.dsi.fastutil.objects.Object2ObjectMap; import net.coderbot.iris.gl.blending.AlphaTest; +import com.gtnewhorizons.angelica.glsm.shader.ShaderDiskCache; import com.gtnewhorizons.angelica.glsm.shader.ShaderType; import com.gtnewhorizons.angelica.glsm.texture.TextureType; import net.coderbot.iris.helpers.Tri; import net.coderbot.iris.pipeline.transform.Patch; import net.coderbot.iris.shaderpack.texture.TextureStage; +import java.util.ArrayList; +import java.util.List; + public abstract class Parameters { public final Patch patch; private final Object2ObjectMap, String> textureMap; @@ -29,6 +33,20 @@ public abstract class Parameters { return textureMap; } + public void appendDiskKey(ShaderDiskCache.Key k) { + k.str(getClass().getName()); + if (textureMap == null) { + k.i(-1); + return; + } + final List rendered = new ArrayList<>(textureMap.size()); + for (Object2ObjectMap.Entry, String> e : textureMap.object2ObjectEntrySet()) { + final Tri tri = e.getKey(); + rendered.add(tri.first() + "\0" + tri.second().name() + "\0" + tri.third().name() + "\0" + e.getValue()); + } + k.sortedStrs(rendered); + } + @Override public int hashCode() { final int prime = 31; diff --git a/src/main/java/net/coderbot/iris/pipeline/transform/parameter/TextureStageParameters.java b/src/main/java/net/coderbot/iris/pipeline/transform/parameter/TextureStageParameters.java index 17edc9d40..f18dc2931 100644 --- a/src/main/java/net/coderbot/iris/pipeline/transform/parameter/TextureStageParameters.java +++ b/src/main/java/net/coderbot/iris/pipeline/transform/parameter/TextureStageParameters.java @@ -2,6 +2,7 @@ package net.coderbot.iris.pipeline.transform.parameter; import it.unimi.dsi.fastutil.objects.Object2ObjectMap; import net.coderbot.iris.gl.blending.AlphaTest; +import com.gtnewhorizons.angelica.glsm.shader.ShaderDiskCache; import com.gtnewhorizons.angelica.glsm.texture.TextureType; import net.coderbot.iris.helpers.Tri; import net.coderbot.iris.pipeline.transform.Patch; @@ -21,6 +22,12 @@ public class TextureStageParameters extends Parameters { return stage; } + @Override + public void appendDiskKey(ShaderDiskCache.Key k) { + super.appendDiskKey(k); + k.str(stage == null ? null : stage.name()); + } + @Override public int hashCode() { final int prime = 31; diff --git a/src/test/java/net/coderbot/iris/pipeline/transform/ShaderTransformerDiskCacheTest.java b/src/test/java/net/coderbot/iris/pipeline/transform/ShaderTransformerDiskCacheTest.java new file mode 100644 index 000000000..3046e0e8d --- /dev/null +++ b/src/test/java/net/coderbot/iris/pipeline/transform/ShaderTransformerDiskCacheTest.java @@ -0,0 +1,132 @@ +package net.coderbot.iris.pipeline.transform; + +import com.gtnewhorizons.angelica.glsm.RenderSystem; +import com.gtnewhorizons.angelica.glsm.shader.ShaderDiskCache; +import com.gtnewhorizons.angelica.glsm.testutil.Reflect; +import com.gtnewhorizons.angelica.glsm.texture.TextureType; +import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; +import net.coderbot.iris.helpers.Tri; +import net.coderbot.iris.pipeline.transform.parameter.ComputeParameters; +import net.coderbot.iris.pipeline.transform.parameter.TextureStageParameters; +import net.coderbot.iris.shaderpack.texture.TextureStage; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.EnumMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +class ShaderTransformerDiskCacheTest { + private static final String VERTEX = "#version 330 core\nvoid main(){ gl_Position = vec4(0.0); }"; + private static final String FRAGMENT = "#version 330 core\nvoid main(){ gl_FragColor = vec4(1.0); }"; + private static final String COMPUTE = String.join("\n", + "#version 330 core", + "layout(local_size_x = 8) in;", + "out vec4 unused;", + "void main(){ unused = vec4(float(gl_GlobalInvocationID.x)); }", + ""); + + @TempDir + Path dir; + + private int savedMaxGlsl; + + @BeforeEach + void setUp() throws Exception { + savedMaxGlsl = Reflect.getStatic(RenderSystem.class, "maxGlslVersion"); + Reflect.setStatic(RenderSystem.class, "maxGlslVersion", 460); + TransformPatcher.clearCache(); + ShaderTransformer.clearCache(); + ShaderTransformer.init(); + ShaderDiskCache.configure(dir, "t"); + } + + @AfterEach + void tearDown() throws Exception { + TransformPatcher.clearCache(); + ShaderTransformer.clearCache(); + Reflect.setStatic(ShaderDiskCache.class, "root", null); + Reflect.setStatic(RenderSystem.class, "maxGlslVersion", savedMaxGlsl); + ShaderTransformer.init(); + } + + private static EnumMap compositeInputs() { + final EnumMap inputs = new EnumMap<>(PatchShaderType.class); + inputs.put(PatchShaderType.VERTEX, VERTEX); + inputs.put(PatchShaderType.GEOMETRY, null); + inputs.put(PatchShaderType.TESS_CONTROL, null); + inputs.put(PatchShaderType.TESS_EVAL, null); + inputs.put(PatchShaderType.FRAGMENT, FRAGMENT); + return inputs; + } + + private static TextureStageParameters compositeParams() { + return new TextureStageParameters(Patch.COMPOSITE, TextureStage.COMPOSITE_AND_FINAL, null); + } + + @Test + void compositeDiskHitSkipsTransform() { + assertNotNull(TransformPatcher.patchComposite(VERTEX, null, FRAGMENT)); + TransformPatcher.clearCache(); + ShaderTransformer.clearCache(); + + ShaderDiskCache.putStrings(ShaderTransformer.diskKey(Patch.COMPOSITE, compositeInputs(), compositeParams()), Map.of("VERTEX", "POISON-V", "FRAGMENT", "POISON-F")); + + final Map out = TransformPatcher.patchComposite(VERTEX, null, FRAGMENT); + assertEquals("POISON-V", out.get(PatchShaderType.VERTEX)); + assertEquals("POISON-F", out.get(PatchShaderType.FRAGMENT)); + } + + @Test + void computeDiskHitSkipsTransform() { + assertNotNull(TransformPatcher.patchCompute("shadowcomp", COMPUTE, null, null)); + TransformPatcher.clearCache(); + ShaderTransformer.clearCache(); + + final EnumMap inputs = new EnumMap<>(PatchShaderType.class); + inputs.put(PatchShaderType.COMPUTE, COMPUTE); + ShaderDiskCache.putStrings(ShaderTransformer.diskKey(Patch.COMPUTE, inputs, new ComputeParameters(Patch.COMPUTE, null, null)), Map.of("COMPUTE", "POISON-C")); + + assertEquals("POISON-C", TransformPatcher.patchCompute("shadowcomp", COMPUTE, null, null)); + } + + @Test + void keyChangesWithGlslVersion() throws Exception { + final String before = ShaderTransformer.diskKey(Patch.COMPOSITE, compositeInputs(), compositeParams()).hex(); + Reflect.setStatic(RenderSystem.class, "maxGlslVersion", 330); + final String after = ShaderTransformer.diskKey(Patch.COMPOSITE, compositeInputs(), compositeParams()).hex(); + assertNotEquals(before, after); + } + + private static Object2ObjectOpenHashMap, String> textureMap(int capacity) { + final Object2ObjectOpenHashMap, String> m = new Object2ObjectOpenHashMap<>(capacity); + m.put(new Tri<>("colortex0", TextureType.TEXTURE_2D, TextureStage.COMPOSITE_AND_FINAL), "a.png"); + m.put(new Tri<>("noisetex", TextureType.TEXTURE_3D, TextureStage.GBUFFERS_AND_SHADOW), "b.png"); + m.put(new Tri<>("gaux1", TextureType.TEXTURE_1D, TextureStage.DEFERRED), "c.png"); + return m; + } + + @Test + void keyIgnoresTextureMapIterationOrder() { + final Object2ObjectOpenHashMap, String> m1 = textureMap(16); + Object2ObjectOpenHashMap, String> m2 = null; + for (int capacity = 32; capacity <= (1 << 20) && m2 == null; capacity <<= 1) { + final Object2ObjectOpenHashMap, String> candidate = textureMap(capacity); + if (!new ArrayList<>(m1.keySet()).equals(new ArrayList<>(candidate.keySet()))) m2 = candidate; + } + assertNotNull(m2, "no capacity produced a different iteration order"); + assertNotEquals(new ArrayList<>(m1.keySet()), new ArrayList<>(m2.keySet())); + assertEquals(m1, m2); + + final String h1 = ShaderTransformer.diskKey(Patch.COMPOSITE, compositeInputs(), new TextureStageParameters(Patch.COMPOSITE, TextureStage.COMPOSITE_AND_FINAL, m1)).hex(); + final String h2 = ShaderTransformer.diskKey(Patch.COMPOSITE, compositeInputs(), new TextureStageParameters(Patch.COMPOSITE, TextureStage.COMPOSITE_AND_FINAL, m2)).hex(); + assertEquals(h1, h2); + } +}