diff --git a/.gitignore b/.gitignore index 6254842..ee80f3d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ main *.pack tool/pack_file *.zip -*.tar \ No newline at end of file +*.tar +*.pyc \ No newline at end of file diff --git a/renpy-parser/generate.py b/renpy-parser/generate.py new file mode 100644 index 0000000..9cb9f43 --- /dev/null +++ b/renpy-parser/generate.py @@ -0,0 +1,42 @@ +import io + +def should_autonewline(line): + return ( + "static_assert" not in line + and "extern" not in line + and (len(line.split()) < 2 or line.split()[1] != '=') # hacky; meh + ) + +def _render(out, lines): + indent = " " + level = 0 + namespace = 0 + for l in lines: + if l and (l[0] == "}" or l[0] == ")"): + level -= 2 + if level < 0: + assert namespace >= 0 + namespace -= 1 + level = 0 + + if len(l) == 0: + out.write("\n") + else: + out.write(indent * level + l + "\n") + + if l and (l[-1] == "{" or l[-1] == "("): + if l.startswith("namespace"): + namespace += 1 + else: + level += 2 + + if level == 0 and l and l[-1] == ";": + if should_autonewline(l): + out.write("\n") + return out + +def renderer(): + out = io.StringIO() + def render(lines): + return _render(out, lines) + return render, out diff --git a/renpy-parser/language/statement.h b/renpy-parser/language/statement.h index fccac9c..5bfcb7f 100644 --- a/renpy-parser/language/statement.h +++ b/renpy-parser/language/statement.h @@ -1,13 +1,61 @@ #include -namespace language::statement { - enum type { +namespace language { + struct option { + char const * const string; + uint32_t statementIndex; + }; + + struct character { + char const * const characterName; + }; + + struct audio { + char const * const path; + }; + + struct image { + char const * const path; + }; + + // statement + + enum struct type { + jump, + menu, + play, + _return, + say, + scene, show, voice, - music, - text, - menu, - jump, + with, + }; + + struct jump { + uint32_t statementIndex; + }; + + struct menu { + uint32_t count; + uint32_t optionIndex; + }; + + struct play { + uint32_t channelIndex; + uint32_t audioIndex; + }; + + struct _return { + }; + + struct say { + uint32_t characterIndex; + uint32_t stringIndex; + }; + + struct scene { + uint32_t imageIndex; }; struct show { @@ -19,39 +67,19 @@ namespace language::statement { uint32_t audioIndex; }; - struct music { - uint32_t channelIndex; - uint32_t audioIndex; - }; - - struct say { - uint32_t characterIndex; - uint32_t stringIndex; - }; - - struct option { - uint32_t stringIndex; - uint32_t statementIndex; - }; - - struct menu { - uint32_t count; - uint32_t optionIndex; - }; - - struct jump { - uint32_t statementIndex; + struct with { }; struct statement { - enum statement_type type; + enum type type; union { + jump jump; + menu menu; + play play; + say say; + scene scene; show show; voice voice; - music music; - say say; - menu menu; - jump jump; }; }; } diff --git a/renpy-parser/lex.py b/renpy-parser/lex.py index eaa1e7e..1c35441 100644 --- a/renpy-parser/lex.py +++ b/renpy-parser/lex.py @@ -179,7 +179,7 @@ def next_token(mem, position): next_position.column += 1 if c >= 128: - print(f"warning: invalid garbage byte {hex(c)} at {position}") + print(f"invalid garbage byte {hex(c)} at {position}", file=sys.stderr) next_position.column = position.column return next_token(mem, next_position) if c == ord('\n'): diff --git a/renpy-parser/parse.py b/renpy-parser/parse.py index 92b4e22..d77c7e1 100644 --- a/renpy-parser/parse.py +++ b/renpy-parser/parse.py @@ -213,16 +213,14 @@ def parse_define(tokens, index): return index, define def parse_label(tokens, index): - name = tokens[index + 0] - if name.type != TT.IDENTIFIER: - raise ParseException("expected identifier", name) + index, lhs = parse_lhs(tokens, index) - colon = tokens[index + 1] + colon = tokens[index] if colon.type != TT.COLON: raise ParseException("expected colon", colon) label = Label( - name = name + name = lhs ) return index + 2, label @@ -253,14 +251,11 @@ def parse_play(tokens, index): return index, play def parse_scene(tokens, index): - name = tokens[index + 0] - if name.type != TT.IDENTIFIER: - raise ParseException("expected identifier", name) - + index, name = parse_lhs(tokens, index) scene = Scene( name = name, ) - return index + 1, scene + return index, scene def parse_with(tokens, index): index, function_call = parse_function_call(tokens, index) @@ -272,11 +267,8 @@ def parse_with(tokens, index): return index, _with def parse_say(tokens, index): - speaker = tokens[index + 0] - if speaker.type != TT.IDENTIFIER: - raise ParseException("expected identifier", name) - - text = tokens[index + 1] + index, speaker = parse_lhs(tokens, index) + text = tokens[index] if text.type != TT.STRING: raise ParseException("expected string", text) @@ -298,15 +290,13 @@ def parse_voice(tokens, index): return index + 1, voice def parse_show(tokens, index): - what = tokens[index + 0] - if what.type != TT.IDENTIFIER: - raise ParseException("expected identifier", path) + index, what = parse_lhs(tokens, index) - at = tokens[index + 1] + at = tokens[index + 0] if at.type != TT.AT: raise ParseException("expected at", at) - transform = tokens[index + 2] + transform = tokens[index + 1] if transform.type != TT.IDENTIFIER: raise ParseException("expected identifier", transform) @@ -314,7 +304,7 @@ def parse_show(tokens, index): what = what, transform = transform ) - return index + 3, show + return index + 2, show def parse_menu(tokens, index): menu = tokens[index + 0] @@ -336,9 +326,7 @@ def parse_menu(tokens, index): continue peek = tokens[index+1] - if token.position.column < menu.position.column: - raise ParseException("invalid block dedent", token) - if token.position.column == menu.position.column: + if token.position.column <= menu.position.column: break if token.type == TT.STRING: @@ -367,9 +355,7 @@ def parse_menu(tokens, index): return index, menu def parse_jump(tokens, index): - target = tokens[index + 0] - if target.type != TT.IDENTIFIER: - raise ParseException("expected identifier", target) + index, target = parse_lhs(tokens, index) jump = Jump( target = target, @@ -395,7 +381,7 @@ def parse_init(tokens, index): continue if token.position.column < init.position.column: - raise ParseException("invalid block dedent", token) + raise ParseException("invalid init block dedent", token) if token.position.column == init.position.column: break index += 1 diff --git a/renpy-parser/transform.py b/renpy-parser/transform.py new file mode 100644 index 0000000..a930b1b --- /dev/null +++ b/renpy-parser/transform.py @@ -0,0 +1,240 @@ +import sys +import lex +import parse +from dataclasses import dataclass +from pprint import pprint +import generate + +@dataclass +class State: + images: list + characters: list + statements: list + menus: list + entries: dict + + images_lookup: dict[str, int] # identifier to image index + characters_lookup: dict[str, int] # identifier to character index + labels_lookup: dict[str, int] # identifier to statement index + audio_lookup: dict[str, int] + string_lookup: dict[str, int] + + global_identifiers: set[str] + +@dataclass +class InternalMenu: + menu: parse.Menu + entry_lookup: dict[str, tuple[int, int]] # string to (entry index, statement index) + entry_index: int + +@dataclass +class InternalJump: + target: tuple + +def lhs_key(lhs): + return tuple(l.lexeme for l in lhs) + +simple_statement_types = { + parse.Jump, + parse.Play, + parse.Return, + parse.Say, + parse.Scene, + parse.Show, + parse.Voice, + parse.With, +} + +def pass1(state, ast): + if type(ast) is parse.Image: + key = lhs_key(ast.name) + assert key not in state.global_identifiers + state.global_identifiers.add(key) + state.images_lookup[key] = len(state.images) + state.images.append(ast) + elif type(ast) is parse.Define: + key = lhs_key(ast.name) + assert key not in state.global_identifiers + state.global_identifiers.add(key) + if type(ast.value) is parse.FunctionCall and ast.value.name.lexeme == b'Character': + state.characters_lookup[key] = len(state.characters) + state.characters.append(ast) + else: + print(f"ignoring define {ast}", file=sys.stderr) + elif type(ast) is parse.Label: + key = lhs_key(ast.name) + assert key not in state.labels_lookup + state.labels_lookup[key] = len(state.statements) + elif type(ast) in {parse.Play, parse.Voice}: + if ast.path.lexeme not in state.audio_lookup: + index = len(state.audio_lookup) + state.audio_lookup[ast.path.lexeme] = index + elif type(ast) is parse.Say: + if ast.text.lexeme not in state.string_lookup: + index = len(state.string_lookup) + state.string_lookup[ast.text.lexeme] = index + elif type(ast) is parse.Menu: + menu_index = len(state.menus) + menu_end_key = (b"__menu_end", menu_index) + entry_lookup = {} + entry_index = len(state.entries) + internal_menu = InternalMenu( + menu = ast, + entry_lookup = entry_lookup, + entry_index = entry_index + ) + state.menus.append(internal_menu) + state.statements.append(internal_menu) + for i, (name, ast_list) in enumerate(ast.entries): + assert name.lexeme not in entry_lookup + entry_lookup[name.lexeme] = entry_index + i + + for name, ast_list in ast.entries: + entry_index = entry_lookup[name.lexeme] + statement_index = len(state.statements) + assert entry_index not in state.entries + state.entries[entry_index] = (name.lexeme, statement_index) + for t in ast_list: + pass1(state, t) + state.statements.append(InternalJump(menu_end_key)) + assert menu_end_key not in state.labels_lookup + state.labels_lookup[menu_end_key] = len(state.statements) + elif type(ast) in simple_statement_types: + pass + else: + assert False, (type(ast), ast) + + if type(ast) in simple_statement_types: + state.statements.append(ast) + +def pass2_statement(state, pc, statement): + if type(statement) is parse.Play: + comment = statement.path.lexeme.decode('utf-8') + audio_index = state.audio_lookup[statement.path.lexeme] + yield f"{{ .type = type::play, .play = {{ .audioIndex = {audio_index} }} }}, // {pc} {comment}" + elif type(statement) is parse.Scene: + key = lhs_key(statement.name) + image_index = state.images_lookup[key] + comment = ".".join(k.decode('utf-8') for k in key) + yield f"{{ .type = type::scene, .scene = {{ .imageIndex = {image_index} }} }}, // {pc} {comment}" + elif type(statement) is parse.With: + print(f"not implemented: {statement}", file=sys.stderr) + pass + elif type(statement) is parse.Voice: + comment = statement.path.lexeme.decode('utf-8') + audio_index = state.audio_lookup[statement.path.lexeme] + yield f"{{ .type = type::voice, .voice = {{ .audioIndex = {audio_index} }} }}, // {pc} {comment}" + elif type(statement) is parse.Say: + key = lhs_key(statement.speaker) + character_index = state.characters_lookup[key] + string_index = state.string_lookup[statement.text.lexeme] + comment = ".".join(k.decode('utf-8') for k in key) + f" \"{statement.text.lexeme.decode('utf-8')}\"" + yield f"{{ .type = type::say, .say = {{ .characterIndex = {character_index}, .stringIndex = {string_index} }} }}, // {pc} {comment}" + elif type(statement) is parse.Show: + key = lhs_key(statement.what) + image_index = state.images_lookup[key] + comment = ".".join(k.decode('utf-8') for k in key) + yield f"{{ .type = type::show, .show = {{ .imageIndex = {image_index} }} }}, // {pc} {comment}" + elif type(statement) is InternalMenu: + count = len(statement.menu.entries) + option_index = statement.entry_index + comment = ", ".join(f"\"{k.lexeme.decode('utf-8')}\"" for k, v in statement.menu.entries) + yield f"{{ .type = type::menu, .menu = {{ .count = {count}, .optionIndex = {option_index} }} }}, // {pc} {comment}" + elif type(statement) is InternalJump: + key = statement.target + statement_index = state.labels_lookup[key] + comment = str(key) + yield f"{{ .type = type::jump, .jump = {{ .statementIndex = {statement_index} }} }}, // {pc} internal jump {comment}" + elif type(statement) is parse.Jump: + key = lhs_key(statement.target) + statement_index = state.labels_lookup[key] + comment = ".".join(k.decode('utf-8') for k in key) + yield f"{{ .type = type::jump, .jump = {{ .statementIndex = {statement_index} }} }}, // {pc} {comment}" + elif type(statement) is parse.Return: + yield f"{{ .type = type::_return }}, // {pc}" + else: + pass + assert False, (type(statement), statement) + +def pass2_statements(state): + yield "const statement statements[] = {" + for pc, statement in enumerate(state.statements): + yield from pass2_statement(state, pc, statement) + yield "};" + +def pass2_strings(state): + yield "char const * const strings[] = {" + for string, i in sorted(state.string_lookup.items(), key=lambda kv: kv[1]): + yield f"\"{string.decode('utf-8')}\", // {i}" + yield "};" + +def pass2_characters(state): + yield "const character characters[] = {" + for i, character in enumerate(state.characters): + character_name, = character.value.args + yield f"{{ .characterName = \"{character_name.lexeme.decode('utf-8')}\" }}, // {i}" + yield "};" + +def pass2_audio(state): + yield "const audio audio[] = {" + for audio, i in sorted(state.audio_lookup.items(), key=lambda kv: kv[1]): + yield f"{{ .path = \"{audio.decode('utf-8')}\" }}, // {i}" + yield "};" + +def pass2_images(state): + yield "const image image[] = {" + for i, image in enumerate(state.images): + yield f"{{ .path = \"{image.path.lexeme.decode('utf-8')}\" }}, // {i}" + yield "};" + +def pass2_options(state): + yield "const option options[] = {" + for i, (lexeme, statement_index) in sorted(state.entries.items(), key=lambda kv: kv[0]): + yield f"{{ .string = \"{lexeme.decode('utf-8')}\", .statementIndex = {statement_index} }}, // {i}" + yield "};" + +def pass2(state): + yield "#include \"statement.h\"" + yield "" + yield "namespace language {" + yield from pass2_strings(state) + yield from pass2_characters(state) + yield from pass2_audio(state) + yield from pass2_images(state) + yield from pass2_options(state) + yield from pass2_statements(state) + yield "}" + +def main(): + with open(sys.argv[1], 'rb') as f: + mem = memoryview(f.read()) + + tokens = list(lex.tokenize(mem)) + state = State( + images = list(), + characters = list(), + statements = list(), + menus = list(), + entries = dict(), + images_lookup = dict(), + characters_lookup = dict(), + labels_lookup = dict(), + audio_lookup = dict(), + string_lookup = dict(), + global_identifiers = set(), + ) + try: + ast_list = list(parse.parse_all(tokens)) + except parse.ParseException as e: + print(e, e.token) + raise + + for t in ast_list: + pass1(state, t) + + render, out = generate.renderer() + render(pass2(state)) + sys.stdout.write(out.getvalue()) + +if __name__ == "__main__": + main()