From 33c2bd0c945c700c3d7cbb6c792a0862ed0bb438 Mon Sep 17 00:00:00 2001 From: Skydark Chen Date: Sat, 14 Nov 2015 13:48:53 +0800 Subject: [PATCH] Add unicode support for zenscripts The unicode characters can be recognized in strings and comments. The encoding of scripts should be utf-8. --- .../src/main/java/minetweaker/runtime/MTTweaker.java | 2 +- .../main/java/stanhebben/zenscript/parser/NFA.java | 6 ++++++ .../java/stanhebben/zenscript/parser/TokenStream.java | 11 ++++++++--- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/MineTweaker3-API/src/main/java/minetweaker/runtime/MTTweaker.java b/MineTweaker3-API/src/main/java/minetweaker/runtime/MTTweaker.java index 324c138..1316c13 100644 --- a/MineTweaker3-API/src/main/java/minetweaker/runtime/MTTweaker.java +++ b/MineTweaker3-API/src/main/java/minetweaker/runtime/MTTweaker.java @@ -128,7 +128,7 @@ public class MTTweaker implements ITweaker { while (script.next()) { Reader reader = null; try { - reader = new InputStreamReader(new BufferedInputStream(script.open())); + reader = new InputStreamReader(new BufferedInputStream(script.open()), "UTF-8"); String filename = script.getName(); String className = extractClassName(filename); diff --git a/ZenScript/src/main/java/stanhebben/zenscript/parser/NFA.java b/ZenScript/src/main/java/stanhebben/zenscript/parser/NFA.java index 8cf6ef9..c461dd0 100644 --- a/ZenScript/src/main/java/stanhebben/zenscript/parser/NFA.java +++ b/ZenScript/src/main/java/stanhebben/zenscript/parser/NFA.java @@ -21,6 +21,10 @@ public class NFA { public static final int NOFINAL = Integer.MIN_VALUE; public static final int EPSILON = Integer.MIN_VALUE + 1; + /* UNICODE_ESCAPE should be between 128 and 256, so unicode chars can be + matched by regexp like '.' and '[^a]' and not by '[a-zA-Z]' */ + public static final int UNICODE_ESCAPE = 256; + private final NFAState initial; private HashMap converted; @@ -318,6 +322,8 @@ public class NFA { /* Processes a single character */ private int processChar(CharStream stream) { if (stream.optional('\\')) { + if (stream.optional('u')) + return UNICODE_ESCAPE; if (stream.optional('e')) return -1; if (stream.optional('r')) diff --git a/ZenScript/src/main/java/stanhebben/zenscript/parser/TokenStream.java b/ZenScript/src/main/java/stanhebben/zenscript/parser/TokenStream.java index ca3ca1a..21145e0 100644 --- a/ZenScript/src/main/java/stanhebben/zenscript/parser/TokenStream.java +++ b/ZenScript/src/main/java/stanhebben/zenscript/parser/TokenStream.java @@ -212,12 +212,17 @@ public class TokenStream implements Iterator { StringBuilder value = new StringBuilder(); int tLine = line; int tLineOffset = lineOffset; - while (dfa.transitions[state].containsKey(nextChar)) { - value.append((char) nextChar); - state = dfa.transitions[state].get(nextChar); + int _fakeNextChar = nextChar > 127 ? NFA.UNICODE_ESCAPE : nextChar; + while (dfa.transitions[state].containsKey(_fakeNextChar)) { + if (nextChar < 0) + value.append((char) nextChar); + else + value.appendCodePoint(nextChar); + state = dfa.transitions[state].get(_fakeNextChar); line = reader.line; lineOffset = reader.lineOffset; nextChar = reader.read(); + _fakeNextChar = nextChar > 127 ? NFA.UNICODE_ESCAPE : nextChar; } if (line < 0)