tools/parse: add parsers for pokemon data

tools/generate/collision_tile_ids.py

@@ -8,7 +8,7 @@ def includes_header():
     yield ''
 
 def extern_collision_tile_ids():
-    for name, index in sorted_tilesets_constants_list():
+    for index, name in sorted_tilesets_constants_list():
         tileset_header = parse.tileset_headers_list()[index]
         coll_path = tileset_header.coll()
         tile_ids = parse.tileset_collision_tile_ids_list()[coll_path]
@@ -20,7 +20,7 @@ def generate_header():
     render(extern_collision_tile_ids())
     return out
 
-def collision_array(name, index):
+def collision_array(index, name):
     tileset_header = parse.tileset_headers_list()[index]
     coll_path = tileset_header.coll()
     tile_ids = parse.tileset_collision_tile_ids_list()[coll_path]
@@ -32,8 +32,8 @@ def collision_array(name, index):
     yield "};"
 
 def collision_tile_ids():
-    for name, index in sorted_tilesets_constants_list():
+    for index, name in sorted_tilesets_constants_list():
-        yield from collision_array(name, index)
+        yield from collision_array(index, name)
 
 def includes_source():
     yield '#include <cstdint>'

tools/generate/maps.py

@@ -24,13 +24,13 @@ from parse.map.headers import Connection
 directions = sorted(['north', 'south', 'east', 'west'])
 
 def sorted_map_constants_list():
-    return sorted(parse.map_constants_list().items(), key=default_sort)
+    return sorted(enumerate(parse.map_constants_list()), key=default_sort)
 
 def sorted_map_headers():
     map_constants_list = sorted_map_constants_list()
     map_headers_dict = dict((map_header.name2, map_header) for map_header in parse.map_headers())
     return (
-        map_headers_dict[map_name2] for map_name2, _ in map_constants_list
+        map_headers_dict[map_name2] for _, map_name2 in map_constants_list
         if map_name2 in map_headers_dict
         # e.g CERULEAN_TRASHED_HOUSE_COPY has no map header
     )

tools/generate/sort.py

@@ -1,5 +1,9 @@
-from operator import itemgetter
+def by_index(l):
+    assert type(l[0]) == int, l
+    return l[0]
+
+def by_name(l):
+    assert type(l[1]) == str, l
+    return l[1]
 
-by_name = itemgetter(0)
-by_index = itemgetter(1)
 default_sort = by_name

tools/generate/sprites.py

@@ -14,14 +14,14 @@ from generate.binary import binary_res, start_size_value
 from generate.generate import renderer
 
 def sorted_sprite_constants_list():
-    return sorted(parse.spritesheet_constants_list().items(), key=default_sort)
+    return sorted(enumerate(parse.spritesheet_constants_list()), key=default_sort)
 
 def includes_header():
     yield '#pragma once'
     yield ''
     yield '#include "../start_size.hpp"'
     yield ''
-    for name, index in sorted_sprite_constants_list():
+    for index, name in sorted_sprite_constants_list():
         if name == 'SPRITE_NONE':
             continue
         assert index != 0, index
@@ -43,7 +43,7 @@ def struct_spritesheet_t():
     _sorted_sprite_constants_list = list(sorted_sprite_constants_list())
     sprite_names = (
         f"{sprite_name(name)},"
-        for name, _ in _sorted_sprite_constants_list
+        for _, name in _sorted_sprite_constants_list
     )
     return [
         "struct spritesheet_t {",
@@ -68,7 +68,7 @@ def generate_header():
     render(sprites_header())
     return out
 
-def sprite(name, index):
+def sprite(index, name):
     if name == 'SPRITE_NONE':
         # special null sprite
         sprite_path = None
@@ -90,8 +90,8 @@ def sprite(name, index):
 
 def sprites():
     yield "const spritesheet_t spritesheets[] = {"
-    for name, index in sorted_sprite_constants_list():
+    for index, name in sorted_sprite_constants_list():
-        yield from sprite(name, index)
+        yield from sprite(index, name)
     yield "};"
 
 def generate_source():

tools/generate/tilesets.py

@@ -5,15 +5,14 @@ from generate.generate import renderer
 from generate.binary import start_size_value
 
 def sorted_tilesets_constants_list():
-    return sorted(parse.tileset_constants_list().items(), key=default_sort)
+    return sorted(enumerate(parse.tileset_constants_list()), key=default_sort)
 
 def includes_header():
     yield "#pragma once"
     yield ""
     yield '#include "../start_size.hpp"'
     yield ""
-    for tileset_name, _ in sorted_tilesets_constants_list():
+    for tileset_index, _ in sorted_tilesets_constants_list():
-        tileset_index  = parse.tileset_constants_list()[tileset_name]
         tileset_header = parse.tileset_headers_list()[tileset_index]
 
         blockset_path = parse.tileset_gfx_list()[tileset_header.blockset()]
@@ -26,7 +25,7 @@ def struct_tileset_t():
     _sorted_tilesets_constants_list = list(sorted_tilesets_constants_list())
     tileset_names = (
         f"{name.lower()},"
-        for name, _ in _sorted_tilesets_constants_list
+        for _, name in _sorted_tilesets_constants_list
     )
     return [
         "struct tileset_t {",
@@ -52,7 +51,7 @@ def generate_header():
     render(extern_tilesets())
     return out
 
-def blockset_tileset(name, index):
+def blockset_tileset(index, name):
     tileset_header = parse.tileset_headers_list()[index]
 
     blockset_path = parse.tileset_gfx_list()[tileset_header.blockset()]
@@ -76,8 +75,8 @@ def blockset_tileset(name, index):
 
 def tilesets():
     yield "const tileset_t tilesets[] = {"
-    for name, index in sorted_tilesets_constants_list():
+    for index, name in sorted_tilesets_constants_list():
-        yield from blockset_tileset(name, index)
+        yield from blockset_tileset(index, name)
     yield "};"
 
 def includes_source():

tools/parse/generic/constants.py

@@ -3,16 +3,20 @@ from functools import partial
 from parse.generic import tokenize
 
 def flatten(tokens):
-    index = 0
     for t in tokens:
-        assert t[0] == 'const', t
+        if t[0] == 'const':
-        _, (name,) = t
+            _, (name,) = t
-        yield name, index
+            yield name
-        index += 1
+        elif t[0] == 'const_skip':
+            yield None
+        elif t[0] == 'const_def':
+            continue
+        else:
+            assert False, t
 
-tokenize_lines = partial(tokenize.lines, prefix='const ')
+tokenize_lines = partial(tokenize.lines, prefix='const')
 
 def parse(prefix, path):
     path = prefix / path
     with open(path) as f:
-        return dict(flatten(tokenize_lines(f.read().split('\n'))))
+        return list(flatten(tokenize_lines(f.read().split('\n'))))

tools/parse/generic/string.py

@@ -0,0 +1,4 @@
+def parse(s):
+    assert s.startswith('"'), s
+    assert s.endswith('"'), s
+    return s[1:-1]

tools/parse/generic/tokenize.py

@@ -20,10 +20,15 @@ def line(line):
 tokenize_line = line
 
 def lines(lines, prefix=""):
+    accumulator = ""
     for line in filter(bool, lines):
         line = line.strip()
         if line.startswith(prefix):
-            yield tokenize_line(line)
+            if line.endswith('\\'):
+                accumulator += line[:-1].strip()
+            else:
+                yield tokenize_line(accumulator + line)
+                accumulator = ""
 
 def block(line, delim):
     name_args = line.split(delim)

tools/parse/move/names.py

@@ -0,0 +1,17 @@
+# similar to parse.pokemon.names
+
+from functools import partial
+
+from parse.generic import tokenize
+from parse.generic import string
+
+lines = partial(tokenize.lines, prefix="li ")
+
+def flatten(tokens):
+    for _, (s,) in tokens:
+        yield string.parse(s)
+
+def parse(prefix):
+    path = prefix / "data/moves/names.asm"
+    with open(path) as f:
+        return list(flatten(lines(f.read().split('\n'))))

tools/parse/pic.py

@@ -0,0 +1,20 @@
+# similar to spritesheet/gfx.py
+
+from parse.generic import tokenize
+from parse.generic.flatten import flatten
+
+def tokenize_lines(lines):
+    for line in lines:
+        if '::' in line:
+            yield tokenize.block(line, delim='::')
+
+def parse(prefix):
+    path = prefix / 'gfx/pics.asm'
+    with open(path) as f:
+        tokens = tokenize_lines(f.read().split('\n'))
+        l = list(flatten(tokens,
+                         endings=['Front', 'Back', 'Pic'],
+                         base_path='gfx/'))
+        d = dict(l)
+        assert len(l) == len(d)
+        return d

tools/parse/pokemon.py

@@ -1,79 +0,0 @@
-# gfx/pics.asm:
-"""
-SquirtlePicFront::     INCBIN "gfx/pokemon/front/squirtle.pic"
-SquirtlePicBack::      INCBIN "gfx/pokemon/back/squirtleb.pic"
-"""
-
-# data/pokemon/base_stats/*.asm
-"""
-	db DEX_TAUROS ; pokedex id
-
-	db  75, 100,  95, 110,  70
-	;   hp  atk  def  spd  spc
-
-	db NORMAL, NORMAL ; type
-	db 45 ; catch rate
-	db 211 ; base exp
-
-	INCBIN "gfx/pokemon/front/tauros.pic", 0, 1 ; sprite dimensions
-	dw TaurosPicFront, TaurosPicBack
-
-	db TACKLE, NO_MOVE, NO_MOVE, NO_MOVE ; level 1 learnset
-	db GROWTH_SLOW ; growth rate
-
-	; tm/hm learnset
-	tmhm TOXIC,        HORN_DRILL,   BODY_SLAM,    TAKE_DOWN,    DOUBLE_EDGE,  \
-	     ICE_BEAM,     BLIZZARD,     HYPER_BEAM,   RAGE,         THUNDERBOLT,  \
-	     THUNDER,      EARTHQUAKE,   FISSURE,      MIMIC,        DOUBLE_TEAM,  \
-	     BIDE,         FIRE_BLAST,   SKULL_BASH,   REST,         SUBSTITUTE,   \
-	     STRENGTH
-	; end
-
-	db 0 ; padding
-"""
-
-# ./data/pokemon/evos_moves.asm
-# ordered by pokemon_constants
-"""
-EvosMovesPointerTable:
-        table_width 2, EvosMovesPointerTable
-        dw RhydonEvosMoves
-        dw KangaskhanEvosMoves
-        dw NidoranMEvosMoves
-        dw ClefairyEvosMoves
-
-....
-
-OddishEvosMoves:
-; Evolutions
-	db EV_LEVEL, 21, GLOOM
-	db 0
-; Learnset
-	db 15, POISONPOWDER
-	db 17, STUN_SPORE
-	db 19, SLEEP_POWDER
-	db 24, ACID
-	db 33, PETAL_DANCE
-	db 46, SOLARBEAM
-	db 0
-"""
-
-# constants/pokemon_constants.asm
-"""
-        const_def
-        const NO_MON             ; $00
-        const RHYDON             ; $01
-        const KANGASKHAN         ; $02
-        const NIDORAN_M          ; $03
-"""
-
-# data/pokemon/dex_entries.asm
-
-# data/pokemon/dex_order.asm
-
-# data/pokemon/names.asm
-
-# data/wild/grass_water.asm
-#   WildDataPointers
-# data/wild/maps/Route1.asm
-#   Route1WildMons

tools/parse/pokemon/base_stats.py

@@ -0,0 +1,70 @@
+from dataclasses import dataclass
+from operator import itemgetter
+from functools import partial
+
+from parse.generic import tokenize
+from parse.generic import string
+from parse.generic import number
+
+lines = partial(tokenize.lines)
+
+def filter_lines(lines):
+    for line in lines:
+        if line[0] in {"db", "dw", "tmhm"}:
+            yield line
+
+@dataclass
+class StatValues:
+    hit_points: int
+    attack: int
+    defense: int
+    speed: int
+    special: int
+
+@dataclass
+class BaseStats:
+    pokedex_id: str
+    stat_values: StatValues
+    types: tuple[str, str]
+    catch_rate: int
+    base_exp: int
+    pic_font_back: tuple[str, str]
+    level_1_learnset: tuple[str, str, str, str]
+    growth_rate: str
+    tmhm: list[str]
+            
+def parse_base_stat(lines):
+    pokedex_id, \
+    stat_values, \
+    types, \
+    catch_rate, \
+    base_exp, \
+    pic_front_back, \
+    level_1_learnset, \
+    growth_rate, \
+    tmhm, \
+    padding = map(itemgetter(1), lines)
+    print(stat_values)
+    return BaseStats(
+        *pokedex_id,
+        StatValues(*map(number.parse, stat_values)),
+        types,
+        number.parse(*catch_rate),
+        number.parse(*base_exp),
+        tuple(pic_front_back),
+        tuple(level_1_learnset),
+        *growth_rate,
+        tmhm,
+    )
+
+
+def parse(path):
+    with open(path) as f:
+        token_lines = filter_lines(lines(f.read().split('\n')))
+        return parse_base_stat(token_lines)
+
+def parse_all(prefix):
+    base_path = prefix / 'data/pokemon/base_stats'
+    paths = [p for p in base_path.iterdir() if p.is_file()]
+    # order is pokedex order, not constant order
+    return [parse(path) for path in paths]

tools/parse/pokemon/constants.py

@@ -1,4 +1,4 @@
 from functools import partial
 from parse.generic import constants
 
-partial(constants.parse, path='constants/pokemon_constants.asm')
+parse = partial(constants.parse, path='constants/pokemon_constants.asm')

tools/parse/pokemon/evos_moves.py

@@ -0,0 +1,117 @@
+import builtins
+from dataclasses import dataclass
+from itertools import chain
+
+from parse.generic import tokenize
+from parse.generic import number
+
+lines = tokenize.lines
+
+def is_label(token_line):
+    return token_line[0].endswith(':')
+
+def is_data(token_line):
+    return token_line[0] == 'db' or token_line[0] == 'dw'
+
+Label = object()
+Data = object()
+
+def event(token_line):
+    if is_label(token_line):
+        label0, = token_line
+        yield Label, label0.split(':')[0]
+    elif is_data(token_line):
+        _, args = token_line
+        yield Data, args
+    else:
+        return
+
+@dataclass
+class EvosMoves:
+    label: str
+    evolutions: list
+    learnset: list
+
+    def __init__(self):
+        self.label = None
+        self.evolutions = []
+        self.learnset = []
+
+def is_evolution(tokens):
+    return tokens[0].startswith('EV_')
+
+def is_learnset_entry(tokens):
+    return len(tokens) == 2 # hmm... hack? maybe not
+
+def parse_ev(tokens):
+    ev_type, *rest = tokens
+    if ev_type == 'EV_ITEM':
+        item_name, level_requirement, pokemon_name = rest
+        return item_name, number.parse(level_requirement), pokemon_name
+    elif ev_type == 'EV_LEVEL' or ev_type == 'EV_TRADE':
+        level_requirement, pokemon_name = rest
+        return number.parse(level_requirement), pokemon_name
+    else:
+        assert False, ev_type
+
+def parse_pointer_table(type_args, ix):
+    pointer_table = []
+    while ix < len(type_args):
+        type, args = type_args[ix]
+        if type is Label:
+            break
+        elif type is Data:
+            evos_moves_label, = args
+            pointer_table.append(evos_moves_label)
+        else:
+            assert False, type_args
+        ix += 1
+    return ix, pointer_table
+
+def parse_learnset_entry(args):
+    level_requirement, move_name = args
+    return number.parse(level_requirement), move_name
+
+def build_tables(tokens):
+    evos_moves = EvosMoves()
+    pointer_table = []
+
+    type_args = list(chain.from_iterable(map(event, tokens)))
+    ix = 0
+    while ix < len(type_args):
+        type, args = type_args[ix]
+        if type is Label:
+            label = args
+            assert builtins.type(label) is str
+            if label == 'EvosMovesPointerTable':
+                ix, pointer_table = parse_pointer_table(type_args, ix + 1)
+            if evos_moves.label is not None:
+                yield evos_moves
+                evos_moves = EvosMoves()
+            else:
+                assert evos_moves.evolutions == [], evos_moves
+                assert evos_moves.learnset == [], evos_moves
+            evos_moves.label = label
+        elif type is Data:
+            if is_evolution(args):
+                evos_moves.evolutions.append(parse_ev(args))
+            elif is_learnset_entry(args):
+                evos_moves.learnset.append(parse_learnset_entry(args))
+            elif args == ['0']:
+                pass # do nothing
+            else:
+                assert False, (type, args)
+        else:
+            assert False, (type, args)
+
+        ix += 1
+
+
+def parse(prefix):
+    path = prefix / "data/pokemon/evos_moves.asm"
+    with open(path) as f:
+        return list(build_tables(lines(f.read().split('\n'))))
+
+from pathlib import Path
+from pprint import pprint
+pprint(parse(Path("../pokered")))

tools/parse/pokemon/names.py

@@ -0,0 +1,17 @@
+# similar to parse.move.names
+
+from functools import partial
+
+from parse.generic import tokenize
+from parse.generic import string
+
+lines = partial(tokenize.lines, prefix="db ")
+
+def flatten(tokens):
+    for _, (s,) in tokens:
+        yield string.parse(s).rstrip('@')
+
+def parse(prefix):
+    path = prefix / "data/pokemon/names.asm"
+    with open(path) as f:
+        return list(flatten(lines(f.read().split('\n'))))

tools/parse/spritesheet/gfx.py

@@ -1,3 +1,5 @@
+# similar to pic.py
+
 from parse.generic import tokenize
 from parse.generic.flatten import flatten