Ping data compression (#8169)

src/main/java/net/minecraftforge/network/ServerStatusPing.java

@@ -1,6 +1,6 @@
 /*
  * Minecraft Forge
- * Copyright (c) 2016-2021.
+ * Copyright (c) 2016-2022.
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -21,11 +21,13 @@ package net.minecraftforge.network;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.gson.JsonArray;
-import com.google.gson.JsonDeserializationContext;
 import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
-import com.google.gson.JsonSerializationContext;
 import com.google.gson.JsonSyntaxException;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import net.minecraft.ResourceLocationException;
+import net.minecraft.network.FriendlyByteBuf;
 import net.minecraft.util.GsonHelper;
 import net.minecraft.resources.ResourceLocation;
 import net.minecraftforge.fml.IExtensionPoint;
@@ -35,16 +37,21 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.function.Supplier;
 
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
 
 /**
+ * Represents additional data sent by FML when a server is pinged.
+ * Previous versions used the following format:
+ * <pre>{@code
  * {
- *      "fmlNetworkVersion" : FMLNETVERSION,
- *      "channels": [
+ *     "fmlNetworkVersion" : FMLNETVERSION,
+ *     "channels": [
  *          {
  *              "res": "fml:handshake",
  *              "version": "1.2.3.4",
@@ -58,21 +65,47 @@ import java.util.stream.StreamSupport;
  *          }
  *     ]
  * }
+ * }</pre>
+ * <p>
+ * Due to size of the ping packet (32767 UTF-16 code points of JSON data) this could exceed this limit and
+ * cause issues. To work around this, a truncation mechanism was introduced, to heuristically truncate the size of the
+ * data, at the expense of making the compatibility info on the server screen inaccurate.
+ *
+ * <p>
+ * Modern versions will send binary data, which is encoded in a custom format optimized for UTF-16 code point count.
+ * See {@link Serializer#encodeOptimized(ByteBuf)} and {@link Serializer#decodeOptimized(String)}.
+ * Essentially 15 bits of binary data are encoded into every UTF-16 code point. The resulting string is then stored in
+ * the "d" property of the resulting JSON.
+ *
+ * <p>
+ * For the format of the binary data see {@link Serializer#serialize(ServerStatusPing)}.
+ *
+ * <p>
+ * The "channels" and "mods" properties are retained for backwards compatibility,
+ * but left empty. A client that cannot read the old format would not be able to connect anyways, but the properties
+ * must exist to not cause exceptions.
+ *
+ * <pre>{@code
+ * {
+ *     "fmlNetworkVersion": FMLNETVERSION,
+ *     "channels": [],
+ *     "mods": [],
+ *     "d": "&lt;binary data&gt;"
+ * }
+ * }</pre>
  *
  */
 public class ServerStatusPing
 {
     private static final Logger LOGGER = LogManager.getLogger();
-    private static final int CHANNEL_TRUNCATE_LIMIT = 150;
-    private static final int MOD_TRUNCATE_LIMIT = 150;
-    private static volatile boolean warnedAboutTruncation = false;
 
-    private transient Map<ResourceLocation, Pair<String, Boolean>> channels;
-    private transient Map<String, String> mods;
-    private transient int fmlNetworkVer;
-    private transient boolean truncated;
+    private final transient Map<ResourceLocation, Pair<String, Boolean>> channels;
+    private final transient Map<String, String> mods;
+    private final transient int fmlNetworkVer;
+    private final transient boolean truncated;
 
-    public ServerStatusPing() {
+    public ServerStatusPing()
+    {
         this.channels = NetworkRegistry.buildChannelVersionsForListPing();
         this.mods = new HashMap<>();
         ModList.get().forEachModContainer((modid, mc) ->
@@ -91,9 +124,56 @@ public class ServerStatusPing
         this.truncated = truncated;
     }
 
-    public static class Serializer {
-        public static ServerStatusPing deserialize(JsonObject forgeData, JsonDeserializationContext ctx) {
-            try {
+    @Override
+    public String toString()
+    {
+        return "FMLStatusPing{" +
+                "channels=" + channels +
+                ", mods=" + mods +
+                ", fmlNetworkVer="
+                + fmlNetworkVer +
+                ", truncated=" + truncated +
+                '}';
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (!(o instanceof ServerStatusPing that)) return false;
+        return fmlNetworkVer == that.fmlNetworkVer && channels.equals(that.channels) && mods.equals(that.mods);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(channels, mods, fmlNetworkVer);
+    }
+
+    private List<Map.Entry<ResourceLocation, Pair<String, Boolean>>> getChannelsForMod(String modId)
+    {
+        return channels.entrySet().stream()
+                .filter(c -> c.getKey().getNamespace().equals(modId))
+                .toList();
+    }
+
+    private List<Map.Entry<ResourceLocation, Pair<String, Boolean>>> getNonModChannels()
+    {
+        return channels.entrySet().stream()
+                .filter(c -> !mods.containsKey(c.getKey().getNamespace()))
+                .toList();
+    }
+
+    public static class Serializer
+    {
+        public static ServerStatusPing deserialize(JsonObject forgeData)
+        {
+            try
+            {
+                if (forgeData.has("d"))
+                {
+                    return deserializeOptimized(forgeData);
+                }
                 final Map<ResourceLocation, Pair<String, Boolean>> channels = StreamSupport.stream(GsonHelper.getAsJsonArray(forgeData, "channels").spliterator(), false).
                         map(JsonElement::getAsJsonObject).
                         collect(Collectors.toMap(jo -> new ResourceLocation(GsonHelper.getAsString(jo, "res")),
@@ -107,60 +187,240 @@ public class ServerStatusPing
                 final int remoteFMLVersion = GsonHelper.getAsInt(forgeData, "fmlNetworkVersion");
                 final boolean truncated = GsonHelper.getAsBoolean(forgeData, "truncated", false);
                 return new ServerStatusPing(channels, mods, remoteFMLVersion, truncated);
-            } catch (JsonSyntaxException e) {
+            }
+            catch (JsonSyntaxException | IndexOutOfBoundsException | ResourceLocationException e)
+            {
                 LOGGER.debug(NetworkConstants.NETWORK, "Encountered an error parsing status ping data", e);
                 return null;
             }
         }
 
-        public static JsonObject serialize(ServerStatusPing forgeData, JsonSerializationContext ctx) {
-            JsonObject obj = new JsonObject();
-            JsonArray channels = new JsonArray();
-            boolean truncated = forgeData.channels.size() > CHANNEL_TRUNCATE_LIMIT || forgeData.mods.size() > MOD_TRUNCATE_LIMIT;
-            if (truncated && !warnedAboutTruncation)
+
+        public static JsonObject serialize(ServerStatusPing forgeData)
+        {
+            // The following techniques are used to keep the size down:
+            // 1. Try and group channels by ModID, this relies on the assumption that a mod "examplemod" uses a channel
+            //    like "examplemod:network". In that case only the "path" of the ResourceLocation is written
+            // 2. Avoid sending IGNORESERVERONLY in plain text, instead use a flag (if set, no version string is sent)
+            //
+            // The size can be estimated as follows (assuming there are no non-mod network channels)
+            // bytes = 2
+            //          + mod_count * (avg_mod_id_length + avg_mod_version_length + 1)
+            //          + (mod_count * avg_channel_count_per_mod) * (avg_mod_channel_length + avg_mod_channel_version_length + 1)
+            //          + 1
+            // for 600 mods with an average ModID and channel length of 20, an average channel of 1 per mod and an
+            // average version length of 5 this turns out to be 31203 bytes, which easily fits into the upper limit of
+            // roughly 60000 bytes. As such it is estimated that the upper limit will never be reached.
+            // we still check though and potentially truncate the list
+
+            var reachedSizeLimit = false;
+            var buf = new FriendlyByteBuf(Unpooled.buffer());
+            buf.writeBoolean(false); // placeholder for whether we are truncating
+            buf.writeShort(forgeData.mods.size()); // short so that we can replace it later in case of truncation
+            int writtenCount = 0;
+            for (var modEntry : forgeData.mods.entrySet())
             {
-                warnedAboutTruncation = true;
-                LOGGER.warn("Heuristically truncating mod and/or impl channel list in server status ping packet. Compatibility report " +
-                        "in the multiplayer screen may be inaccurate.");
+                var isIgnoreServerOnly = modEntry.getValue().equals(NetworkConstants.IGNORESERVERONLY);
+
+                var channelsForMod = forgeData.getChannelsForMod(modEntry.getKey());
+                var channelSizeAndVersionFlag = channelsForMod.size() << 1;
+                if (isIgnoreServerOnly)
+                {
+                    channelSizeAndVersionFlag |= VERSION_FLAG_IGNORESERVERONLY;
+                }
+                buf.writeVarInt(channelSizeAndVersionFlag);
+
+                buf.writeUtf(modEntry.getKey());
+                if (!isIgnoreServerOnly)
+                {
+                    buf.writeUtf(modEntry.getValue());
+                }
+
+                // write the channels for this mod, if any
+                for (var entry : channelsForMod)
+                {
+                    buf.writeUtf(entry.getKey().getPath());
+                    buf.writeUtf(entry.getValue().getLeft());
+                    buf.writeBoolean(entry.getValue().getRight());
+                }
+
+                writtenCount++;
+
+                if (buf.readableBytes() >= 60000)
+                {
+                    reachedSizeLimit = true;
+                    break;
+                }
             }
 
-            forgeData.channels.entrySet().stream().limit(CHANNEL_TRUNCATE_LIMIT).forEach(entry -> {
-                ResourceLocation namespace = entry.getKey();
-                Pair<String, Boolean> version = entry.getValue();
-                JsonObject mi = new JsonObject();
-                mi.addProperty("res", namespace.toString());
-                mi.addProperty("version", version.getLeft());
-                mi.addProperty("required", version.getRight());
-                channels.add(mi);
-            });
+            if (!reachedSizeLimit)
+            {
+                // write any channels that don't match up with a ModID.
+                var nonModChannels = forgeData.getNonModChannels();
+                buf.writeVarInt(nonModChannels.size());
+                for (var entry : nonModChannels)
+                {
+                    buf.writeResourceLocation(entry.getKey());
+                    buf.writeUtf(entry.getValue().getLeft());
+                    buf.writeBoolean(entry.getValue().getRight());
+                }
+            }
+            else
+            {
+                buf.setShort(1, writtenCount);
+                buf.writeVarInt(0);
+            }
 
-            obj.add("channels", channels);
+            buf.setBoolean(0, reachedSizeLimit);
 
-            JsonArray modTestValues = new JsonArray();
-            forgeData.mods.entrySet().stream().limit(MOD_TRUNCATE_LIMIT).forEach(entry -> {
-                String modId = entry.getKey();
-                String value = entry.getValue();
-                JsonObject mi = new JsonObject();
-                mi.addProperty("modId", modId);
-                mi.addProperty("modmarker", value);
-                modTestValues.add(mi);
-            });
-            obj.add("mods", modTestValues);
+            var obj = new JsonObject();
             obj.addProperty("fmlNetworkVersion", forgeData.fmlNetworkVer);
-            obj.addProperty("truncated", truncated);
+            obj.addProperty("d", encodeOptimized(buf));
+
+            // add dummy properties, so old versions do not crash when deserializing
+            obj.add("channels", new JsonArray());
+            obj.add("mods", new JsonArray());
+            // legacy versions see truncated lists, modern versions ignore this truncated flag (binary data has its own)
+            obj.addProperty("truncated", true);
             return obj;
         }
+
+        private static final int VERSION_FLAG_IGNORESERVERONLY = 0b1;
+
+        private static ServerStatusPing deserializeOptimized(JsonObject forgeData)
+        {
+            int remoteFMLVersion = GsonHelper.getAsInt(forgeData, "fmlNetworkVersion");
+            var buf = new FriendlyByteBuf(decodeOptimized(GsonHelper.getAsString(forgeData, "d")));
+
+            boolean truncated;
+            Map<ResourceLocation, Pair<String, Boolean>> channels;
+            Map<String, String> mods;
+
+            try
+            {
+                truncated = buf.readBoolean();
+                var modsSize = buf.readUnsignedShort();
+                mods = new HashMap<>();
+                channels = new HashMap<>();
+                for (var i = 0; i < modsSize; i++) {
+                    var channelSizeAndVersionFlag = buf.readVarInt();
+                    var channelSize = channelSizeAndVersionFlag >>> 1;
+                    var isIgnoreServerOnly = (channelSizeAndVersionFlag & VERSION_FLAG_IGNORESERVERONLY) != 0;
+                    var modId = buf.readUtf();
+                    var modVersion = isIgnoreServerOnly ? NetworkConstants.IGNORESERVERONLY : buf.readUtf();
+                    for (var i1 = 0; i1 < channelSize; i1++) {
+                        var channelName = buf.readUtf();
+                        var channelVersion = buf.readUtf();
+                        var requiredOnClient = buf.readBoolean();
+                        channels.put(new ResourceLocation(modId, channelName), Pair.of(channelVersion, requiredOnClient));
+                    }
+
+                    mods.put(modId, modVersion);
+                }
+
+                var nonModChannelCount = buf.readVarInt();
+                for (var i = 0; i < nonModChannelCount; i++) {
+                    var channelName = buf.readResourceLocation();
+                    var channelVersion = buf.readUtf();
+                    var requiredOnClient = buf.readBoolean();
+                    channels.put(channelName, Pair.of(channelVersion, requiredOnClient));
+                }
+            }
+            finally
+            {
+                buf.release();
+            }
+
+            return new ServerStatusPing(channels, mods, remoteFMLVersion, truncated);
+        }
+
+        /**
+         * Encode given ByteBuf to a String. This is optimized for UTF-16 Code-Point count.
+         * Supports at most 2^30 bytes in length
+         */
+        private static String encodeOptimized(ByteBuf buf)
+        {
+            var byteLength = buf.readableBytes();
+            var sb = new StringBuilder();
+            sb.append((char) (byteLength & 0x7FFF));
+            sb.append((char) ((byteLength >>> 15) & 0x7FFF));
+
+            int buffer = 0; // we will need at most 8 + 14 = 22 bits of buffer, so an int is enough
+            int bitsInBuf = 0;
+            while (buf.isReadable())
+            {
+                if (bitsInBuf >= 15)
+                {
+                    char c = (char) (buffer & 0x7FFF);
+                    sb.append(c);
+                    buffer >>>= 15;
+                    bitsInBuf -= 15;
+                }
+                var b = buf.readUnsignedByte();
+                buffer |= (int) b << bitsInBuf;
+                bitsInBuf += 8;
+            }
+
+            if (bitsInBuf > 0)
+            {
+                char c = (char) (buffer & 0x7FFF);
+                sb.append(c);
+            }
+            return sb.toString();
+        }
+
+        /**
+         * Decode binary data encoded by {@link #encodeOptimized}
+         */
+        private static ByteBuf decodeOptimized(String s)
+        {
+            var size0 = ((int) s.charAt(0));
+            var size1 = ((int) s.charAt(1));
+            var size = size0 | (size1 << 15);
+
+            var buf = Unpooled.buffer(size);
+
+            int stringIndex = 2;
+            int buffer = 0; // we will need at most 8 + 14 = 22 bits of buffer, so an int is enough
+            int bitsInBuf = 0;
+            while (stringIndex < s.length())
+            {
+                while (bitsInBuf >= 8)
+                {
+                    buf.writeByte(buffer);
+                    buffer >>>= 8;
+                    bitsInBuf -= 8;
+                }
+
+                var c = s.charAt(stringIndex);
+                buffer |= (((int) c) & 0x7FFF) << bitsInBuf;
+                bitsInBuf += 15;
+                stringIndex++;
+            }
+
+            // write any leftovers
+            while (buf.readableBytes() < size)
+            {
+                buf.writeByte(buffer);
+                buffer >>>= 8;
+                bitsInBuf -= 8;
+            }
+            return buf;
+        }
     }
 
-    public Map<ResourceLocation, Pair<String, Boolean>> getRemoteChannels() {
+    public Map<ResourceLocation, Pair<String, Boolean>> getRemoteChannels()
+    {
         return this.channels;
     }
 
-    public Map<String,String> getRemoteModData() {
+    public Map<String,String> getRemoteModData()
+    {
         return mods;
     }
 
-    public int getFMLNetworkVersion() {
+    public int getFMLNetworkVersion()
+    {
         return fmlNetworkVer;
     }