renpy-parser: implement transform

.gitignore

@@ -5,4 +5,5 @@ main
 *.pack
 tool/pack_file
 *.zip
-*.tar
+*.tar
+*.pyc

renpy-parser/generate.py

@@ -0,0 +1,42 @@
+import io
+
+def should_autonewline(line):
+    return (
+        "static_assert" not in line
+        and "extern" not in line
+        and (len(line.split()) < 2 or line.split()[1] != '=') # hacky; meh
+    )
+
+def _render(out, lines):
+    indent = " "
+    level = 0
+    namespace = 0
+    for l in lines:
+        if l and (l[0] == "}" or l[0] == ")"):
+            level -= 2
+            if level < 0:
+                assert namespace >= 0
+                namespace -= 1
+                level = 0
+
+        if len(l) == 0:
+            out.write("\n")
+        else:
+            out.write(indent * level + l + "\n")
+
+        if l and (l[-1] == "{" or l[-1] == "("):
+            if l.startswith("namespace"):
+                namespace += 1
+            else:
+                level += 2
+
+        if level == 0 and l and l[-1] == ";":
+            if should_autonewline(l):
+                out.write("\n")
+    return out
+
+def renderer():
+    out = io.StringIO()
+    def render(lines):
+        return _render(out, lines)
+    return render, out

renpy-parser/language/statement.h

@@ -1,13 +1,61 @@
 #include <stdint.h>
 
-namespace language::statement {
+namespace language {
-  enum type {
+  struct option {
+    char const * const string;
+    uint32_t statementIndex;
+  };
+
+  struct character {
+    char const * const characterName;
+  };
+
+  struct audio {
+    char const * const path;
+  };
+
+  struct image {
+    char const * const path;
+  };
+
+  // statement
+
+  enum struct type {
+    jump,
+    menu,
+    play,
+    _return,
+    say,
+    scene,
     show,
     voice,
-    music,
+    with,
-    text,
+  };
-    menu,
+
-    jump,
+  struct jump {
+    uint32_t statementIndex;
+  };
+
+  struct menu {
+    uint32_t count;
+    uint32_t optionIndex;
+  };
+
+  struct play {
+    uint32_t channelIndex;
+    uint32_t audioIndex;
+  };
+
+  struct _return {
+  };
+
+  struct say {
+    uint32_t characterIndex;
+    uint32_t stringIndex;
+  };
+
+  struct scene {
+    uint32_t imageIndex;
   };
 
   struct show {
@@ -19,39 +67,19 @@ namespace language::statement {
     uint32_t audioIndex;
   };
 
-  struct music {
+  struct with {
-    uint32_t channelIndex;
-    uint32_t audioIndex;
-  };
-
-  struct say {
-    uint32_t characterIndex;
-    uint32_t stringIndex;
-  };
-
-  struct option {
-    uint32_t stringIndex;
-    uint32_t statementIndex;
-  };
-
-  struct menu {
-    uint32_t count;
-    uint32_t optionIndex;
-  };
-
-  struct jump {
-    uint32_t statementIndex;
   };
 
   struct statement {
-    enum statement_type type;
+    enum type type;
     union {
+      jump jump;
+      menu menu;
+      play play;
+      say say;
+      scene scene;
       show show;
       voice voice;
-      music music;
-      say say;
-      menu menu;
-      jump jump;
     };
   };
 }

renpy-parser/lex.py

@@ -179,7 +179,7 @@ def next_token(mem, position):
     next_position.column += 1
 
     if c >= 128:
-        print(f"warning: invalid garbage byte {hex(c)} at {position}")
+        print(f"invalid garbage byte {hex(c)} at {position}", file=sys.stderr)
         next_position.column = position.column
         return next_token(mem, next_position)
     if c == ord('\n'):

renpy-parser/parse.py

@@ -213,16 +213,14 @@ def parse_define(tokens, index):
     return index, define
 
 def parse_label(tokens, index):
-    name = tokens[index + 0]
+    index, lhs = parse_lhs(tokens, index)
-    if name.type != TT.IDENTIFIER:
-        raise ParseException("expected identifier", name)
 
-    colon = tokens[index + 1]
+    colon = tokens[index]
     if colon.type != TT.COLON:
         raise ParseException("expected colon", colon)
 
     label = Label(
-        name = name
+        name = lhs
     )
 
     return index + 2, label
@@ -253,14 +251,11 @@ def parse_play(tokens, index):
     return index, play
 
 def parse_scene(tokens, index):
-    name = tokens[index + 0]
+    index, name = parse_lhs(tokens, index)
-    if name.type != TT.IDENTIFIER:
-        raise ParseException("expected identifier", name)
-
     scene = Scene(
         name = name,
     )
-    return index + 1, scene
+    return index, scene
 
 def parse_with(tokens, index):
     index, function_call = parse_function_call(tokens, index)
@@ -272,11 +267,8 @@ def parse_with(tokens, index):
     return index, _with
 
 def parse_say(tokens, index):
-    speaker = tokens[index + 0]
+    index, speaker = parse_lhs(tokens, index)
-    if speaker.type != TT.IDENTIFIER:
+    text = tokens[index]
-        raise ParseException("expected identifier", name)
-
-    text = tokens[index + 1]
     if text.type != TT.STRING:
         raise ParseException("expected string", text)
 
@@ -298,15 +290,13 @@ def parse_voice(tokens, index):
     return index + 1, voice
 
 def parse_show(tokens, index):
-    what = tokens[index + 0]
+    index, what = parse_lhs(tokens, index)
-    if what.type != TT.IDENTIFIER:
-        raise ParseException("expected identifier", path)
 
-    at = tokens[index + 1]
+    at = tokens[index + 0]
     if at.type != TT.AT:
         raise ParseException("expected at", at)
 
-    transform = tokens[index + 2]
+    transform = tokens[index + 1]
     if transform.type != TT.IDENTIFIER:
         raise ParseException("expected identifier", transform)
 
@@ -314,7 +304,7 @@ def parse_show(tokens, index):
         what = what,
         transform = transform
     )
-    return index + 3, show
+    return index + 2, show
 
 def parse_menu(tokens, index):
     menu = tokens[index + 0]
@@ -336,9 +326,7 @@ def parse_menu(tokens, index):
             continue
         peek = tokens[index+1]
 
-        if token.position.column < menu.position.column:
+        if token.position.column <= menu.position.column:
-            raise ParseException("invalid block dedent", token)
-        if token.position.column == menu.position.column:
             break
 
         if token.type == TT.STRING:
@@ -367,9 +355,7 @@ def parse_menu(tokens, index):
     return index, menu
 
 def parse_jump(tokens, index):
-    target = tokens[index + 0]
+    index, target = parse_lhs(tokens, index)
-    if target.type != TT.IDENTIFIER:
-        raise ParseException("expected identifier", target)
 
     jump = Jump(
         target = target,
@@ -395,7 +381,7 @@ def parse_init(tokens, index):
             continue
 
         if token.position.column < init.position.column:
-            raise ParseException("invalid block dedent", token)
+            raise ParseException("invalid init block dedent", token)
         if token.position.column == init.position.column:
             break
         index += 1

renpy-parser/transform.py

@@ -0,0 +1,240 @@
+import sys
+import lex
+import parse
+from dataclasses import dataclass
+from pprint import pprint
+import generate
+
+@dataclass
+class State:
+    images: list
+    characters: list
+    statements: list
+    menus: list
+    entries: dict
+
+    images_lookup: dict[str, int] # identifier to image index
+    characters_lookup: dict[str, int] # identifier to character index
+    labels_lookup: dict[str, int] # identifier to statement index
+    audio_lookup: dict[str, int]
+    string_lookup: dict[str, int]
+
+    global_identifiers: set[str]
+
+@dataclass
+class InternalMenu:
+    menu: parse.Menu
+    entry_lookup: dict[str, tuple[int, int]] # string to (entry index, statement index)
+    entry_index: int
+
+@dataclass
+class InternalJump:
+    target: tuple
+
+def lhs_key(lhs):
+    return tuple(l.lexeme for l in lhs)
+
+simple_statement_types = {
+    parse.Jump,
+    parse.Play,
+    parse.Return,
+    parse.Say,
+    parse.Scene,
+    parse.Show,
+    parse.Voice,
+    parse.With,
+}
+
+def pass1(state, ast):
+    if type(ast) is parse.Image:
+        key = lhs_key(ast.name)
+        assert key not in state.global_identifiers
+        state.global_identifiers.add(key)
+        state.images_lookup[key] = len(state.images)
+        state.images.append(ast)
+    elif type(ast) is parse.Define:
+        key = lhs_key(ast.name)
+        assert key not in state.global_identifiers
+        state.global_identifiers.add(key)
+        if type(ast.value) is parse.FunctionCall and ast.value.name.lexeme == b'Character':
+            state.characters_lookup[key] = len(state.characters)
+            state.characters.append(ast)
+        else:
+            print(f"ignoring define {ast}", file=sys.stderr)
+    elif type(ast) is parse.Label:
+        key = lhs_key(ast.name)
+        assert key not in state.labels_lookup
+        state.labels_lookup[key] = len(state.statements)
+    elif type(ast) in {parse.Play, parse.Voice}:
+        if ast.path.lexeme not in state.audio_lookup:
+            index = len(state.audio_lookup)
+            state.audio_lookup[ast.path.lexeme] = index
+    elif type(ast) is parse.Say:
+        if ast.text.lexeme not in state.string_lookup:
+            index = len(state.string_lookup)
+            state.string_lookup[ast.text.lexeme] = index
+    elif type(ast) is parse.Menu:
+        menu_index = len(state.menus)
+        menu_end_key = (b"__menu_end", menu_index)
+        entry_lookup = {}
+        entry_index = len(state.entries)
+        internal_menu = InternalMenu(
+            menu = ast,
+            entry_lookup = entry_lookup,
+            entry_index = entry_index
+        )
+        state.menus.append(internal_menu)
+        state.statements.append(internal_menu)
+        for i, (name, ast_list) in enumerate(ast.entries):
+            assert name.lexeme not in entry_lookup
+            entry_lookup[name.lexeme] = entry_index + i
+
+        for name, ast_list in ast.entries:
+            entry_index = entry_lookup[name.lexeme]
+            statement_index = len(state.statements)
+            assert entry_index not in state.entries
+            state.entries[entry_index] = (name.lexeme, statement_index)
+            for t in ast_list:
+                pass1(state, t)
+            state.statements.append(InternalJump(menu_end_key))
+        assert menu_end_key not in state.labels_lookup
+        state.labels_lookup[menu_end_key] = len(state.statements)
+    elif type(ast) in simple_statement_types:
+        pass
+    else:
+        assert False, (type(ast), ast)
+
+    if type(ast) in simple_statement_types:
+        state.statements.append(ast)
+
+def pass2_statement(state, pc, statement):
+    if type(statement) is parse.Play:
+        comment = statement.path.lexeme.decode('utf-8')
+        audio_index = state.audio_lookup[statement.path.lexeme]
+        yield f"{{ .type = type::play, .play = {{ .audioIndex = {audio_index} }} }}, // {pc} {comment}"
+    elif type(statement) is parse.Scene:
+        key = lhs_key(statement.name)
+        image_index = state.images_lookup[key]
+        comment = ".".join(k.decode('utf-8') for k in key)
+        yield f"{{ .type = type::scene, .scene = {{ .imageIndex = {image_index} }} }}, // {pc} {comment}"
+    elif type(statement) is parse.With:
+        print(f"not implemented: {statement}", file=sys.stderr)
+        pass
+    elif type(statement) is parse.Voice:
+        comment = statement.path.lexeme.decode('utf-8')
+        audio_index = state.audio_lookup[statement.path.lexeme]
+        yield f"{{ .type = type::voice, .voice = {{ .audioIndex = {audio_index} }} }}, // {pc} {comment}"
+    elif type(statement) is parse.Say:
+        key = lhs_key(statement.speaker)
+        character_index = state.characters_lookup[key]
+        string_index = state.string_lookup[statement.text.lexeme]
+        comment = ".".join(k.decode('utf-8') for k in key) + f" \"{statement.text.lexeme.decode('utf-8')}\""
+        yield f"{{ .type = type::say, .say = {{ .characterIndex = {character_index}, .stringIndex = {string_index} }} }}, // {pc} {comment}"
+    elif type(statement) is parse.Show:
+        key = lhs_key(statement.what)
+        image_index = state.images_lookup[key]
+        comment = ".".join(k.decode('utf-8') for k in key)
+        yield f"{{ .type = type::show, .show = {{ .imageIndex = {image_index} }} }}, // {pc} {comment}"
+    elif type(statement) is InternalMenu:
+        count = len(statement.menu.entries)
+        option_index = statement.entry_index
+        comment = ", ".join(f"\"{k.lexeme.decode('utf-8')}\"" for k, v in statement.menu.entries)
+        yield f"{{ .type = type::menu, .menu = {{ .count = {count}, .optionIndex = {option_index} }} }}, // {pc} {comment}"
+    elif type(statement) is InternalJump:
+        key = statement.target
+        statement_index = state.labels_lookup[key]
+        comment = str(key)
+        yield f"{{ .type = type::jump, .jump = {{ .statementIndex = {statement_index} }} }}, // {pc} internal jump {comment}"
+    elif type(statement) is parse.Jump:
+        key = lhs_key(statement.target)
+        statement_index = state.labels_lookup[key]
+        comment = ".".join(k.decode('utf-8') for k in key)
+        yield f"{{ .type = type::jump, .jump = {{ .statementIndex = {statement_index} }} }}, // {pc} {comment}"
+    elif type(statement) is parse.Return:
+        yield f"{{ .type = type::_return }}, // {pc}"
+    else:
+        pass
+        assert False, (type(statement), statement)
+
+def pass2_statements(state):
+    yield "const statement statements[] = {"
+    for pc, statement in enumerate(state.statements):
+        yield from pass2_statement(state, pc, statement)
+    yield "};"
+
+def pass2_strings(state):
+    yield "char const * const strings[] = {"
+    for string, i in sorted(state.string_lookup.items(), key=lambda kv: kv[1]):
+        yield f"\"{string.decode('utf-8')}\", // {i}"
+    yield "};"
+
+def pass2_characters(state):
+    yield "const character characters[] = {"
+    for i, character in enumerate(state.characters):
+        character_name, = character.value.args
+        yield f"{{ .characterName = \"{character_name.lexeme.decode('utf-8')}\" }}, // {i}"
+    yield "};"
+
+def pass2_audio(state):
+    yield "const audio audio[] = {"
+    for audio, i in sorted(state.audio_lookup.items(), key=lambda kv: kv[1]):
+        yield f"{{ .path = \"{audio.decode('utf-8')}\" }}, // {i}"
+    yield "};"
+
+def pass2_images(state):
+    yield "const image image[] = {"
+    for i, image in enumerate(state.images):
+        yield f"{{ .path = \"{image.path.lexeme.decode('utf-8')}\" }}, // {i}"
+    yield "};"
+
+def pass2_options(state):
+    yield "const option options[] = {"
+    for i, (lexeme, statement_index) in sorted(state.entries.items(), key=lambda kv: kv[0]):
+        yield f"{{ .string = \"{lexeme.decode('utf-8')}\", .statementIndex = {statement_index} }}, // {i}"
+    yield "};"
+
+def pass2(state):
+    yield "#include \"statement.h\""
+    yield ""
+    yield "namespace language {"
+    yield from pass2_strings(state)
+    yield from pass2_characters(state)
+    yield from pass2_audio(state)
+    yield from pass2_images(state)
+    yield from pass2_options(state)
+    yield from pass2_statements(state)
+    yield "}"
+
+def main():
+    with open(sys.argv[1], 'rb') as f:
+        mem = memoryview(f.read())
+
+    tokens = list(lex.tokenize(mem))
+    state = State(
+        images = list(),
+        characters = list(),
+        statements = list(),
+        menus = list(),
+        entries = dict(),
+        images_lookup = dict(),
+        characters_lookup = dict(),
+        labels_lookup = dict(),
+        audio_lookup = dict(),
+        string_lookup = dict(),
+        global_identifiers = set(),
+    )
+    try:
+        ast_list = list(parse.parse_all(tokens))
+    except parse.ParseException as e:
+        print(e, e.token)
+        raise
+
+    for t in ast_list:
+        pass1(state, t)
+
+    render, out = generate.renderer()
+    render(pass2(state))
+    sys.stdout.write(out.getvalue())
+
+if __name__ == "__main__":
+    main()