initial

binary_image_palette.py

@@ -0,0 +1,78 @@
+import struct
+import sys
+
+from PIL import Image
+
+def round_up_palette_size(palette_size):
+    assert palette_size != 0, (name, palette_size)
+    if palette_size <= 4:
+        return 4
+    elif palette_size <= 16:
+        return 16
+    elif palette_size <= 256:
+        return 256
+    else:
+        assert False, palette_size
+
+def pixels_per_byte(palette_size):
+    if palette_size == 4:
+        return 4
+    elif palette_size == 16:
+        return 2
+    elif palette_size == 256:
+        return 1
+    else:
+        assert False, palette_size
+
+def pack_one_byte(pixels, index, colors, palette_size):
+    color_count = len(colors)
+    num = pixels_per_byte(palette_size)
+    shift = 8 // num
+    byte = 0
+    i = 0
+    while num > 0:
+        px, alpha = pixels[index]
+        if alpha == 0 and color_count < palette_size:
+            px = color_count
+        assert px < palette_size
+        byte |= px << (shift * i)
+        index += 1
+        i += 1
+        num -= 1
+    return [byte], index
+
+def pack_one_texel(pixels, index, colors, palette_size):
+    px, alpha = pixels[index]
+    return
+
+def pack_pixels(pixels, width, height, colors, palette_size):
+    index = 0
+    with open(sys.argv[2], 'wb') as f:
+        while index < (width * height):
+            byte_list, index = pack_texel(pixels, index, colors, palette_size)
+            f.write(bytes(byte))
+
+def pack_palette(colors, palette_size):
+    with open(sys.argv[2], 'wb') as f:
+        for color in colors:
+            out = argb1555(255, *color)
+            f.write(struct.pack('<H', out))
+        if len(colors) < palette_size:
+            # transparent color
+            print('pack transparency at ix', len(colors))
+            out = argb1555(0, 0, 0, 0)
+            f.write(struct.pack('<H', out))
+
+with Image.open(sys.argv[1]) as im:
+    assert im.mode == "P"
+    width, height = im.size
+    colors = list(im.palette.colors)
+    pixels = list(im.convert("PA").getdata())
+
+palette_size = round_up_palette_size(len(colors))
+if sys.argv[2].endswith('.data'):
+    pack_pixels(pixels, width, height, len(colors), palette_size)
+elif sys.argv[2].endswith('.pal'):
+    pack_palette(colors, palette_size)
+else:
+    assert False, sys.argv[2]

fixed_point.py

@@ -0,0 +1,77 @@
+from dataclasses import dataclass
+import string
+
+class FixedPointOverflow(Exception):
+    pass
+
+@dataclass
+class FixedPoint:
+    sign: int
+    value: int
+    point: int
+
+    def to_fixed_point(fp, integer_bits, fraction_bits):
+        point = 1 << fraction_bits
+        value = (fp.value * point) // fp.point
+        if integer_bits is not None:
+            integer_point = 1 << integer_bits
+            integer = value // point
+            if integer >= integer_point:
+                raise FixedPointOverflow((integer, integer_point))
+        return FixedPoint(
+            fp.sign,
+            value,
+            point
+        )
+
+    def to_int(fp):
+        return fp.sign * fp.value
+
+    def to_float(fp):
+        return fp.sign * fp.value / fp.point
+
+def from_float(n):
+    if n == 0.0:
+        sign = 1
+        value = 0
+    else:
+        sign = -1 if n < 0 else 1
+        value = abs(round(n * (2 ** 32)))
+    point = 2 ** 32
+    return FixedPoint(sign, value, point)
+
+assert from_float(0.5).to_float() == 0.5
+assert from_float(1.5).to_fixed_point(16, 16).value == 98304
+
+def parse(s):
+    sign = -1 if s.startswith('-') else 1
+    s = s.removeprefix('-')
+    integer, fraction = s.split('.')
+    assert all(c in string.digits for c in integer), integer
+    assert all(c in string.digits for c in fraction), fraction
+    assert len(integer) > 0 and len(fraction) > 0, s
+    point = 10 ** len(fraction)
+    value = int(integer) * point + int(fraction)
+    return FixedPoint(
+        sign,
+        value,
+        point
+    )
+
+def equal(a, b):
+    epsilon = 0.00001
+    return (a - b) < epsilon
+
+def assert_raises(e, f):
+    try:
+        f()
+    except e:
+        return
+    raise AssertionError(f"expected {str(e)}")
+
+assert parse("1.234").value == 1234
+assert equal(parse("1.234").to_float(), 1.234)
+assert parse("1.234").to_fixed_point(16, 16).value == 80871
+assert_raises(FixedPointOverflow,
+              lambda: parse("2.234").to_fixed_point(1, 16))
+assert parse("2.234").to_fixed_point(2, 16).value == 146407

generate.py

@@ -0,0 +1,35 @@
+import io
+
+def should_autonewline(line):
+    return (
+        "static_assert" not in line
+        and "extern" not in line
+        and (len(line.split()) < 2 or line.split()[1] != '=') # hacky; meh
+    )
+
+def _render(out, lines):
+    indent = " "
+    level = 0
+    for l in lines:
+        if l and (l[0] == "}" or l[0] == ")"):
+            level -= 2
+            assert level >= 0, out.getvalue()
+
+        if len(l) == 0:
+            out.write("\n")
+        else:
+            out.write(indent * level + l + "\n")
+
+        if l and (l[-1] == "{" or l[-1] == "("):
+            level += 2
+
+        if level == 0 and l and l[-1] == ";":
+            if should_autonewline(l):
+                out.write("\n")
+    return out
+
+def renderer():
+    out = io.StringIO()
+    def render(lines):
+        return _render(out, lines)
+    return render, out

parse_material.py

@@ -0,0 +1,47 @@
+from dataclasses import dataclass
+
+@dataclass
+class NewMtl:
+    name: str
+
+@dataclass
+class MapKd:
+    name: str
+
+def parse_material_newmtl(args):
+    name, = args.split()
+    yield NewMtl(name.replace('-', '_').replace('.', '_'))
+
+def parse_material_mapkd(args):
+    name, = args.split()
+    yield MapKd(name)
+
+def parse_mtl_line(line):
+    prefixes = [
+        ('newmtl ', parse_material_newmtl),
+        ('map_Kd ', parse_material_mapkd),
+    ]
+    for prefix, parser in prefixes:
+        if line.startswith(prefix):
+            args = line.removeprefix(prefix)
+            yield from parser(args)
+
+def group_by_material(l):
+    current_material = None
+    for i in l:
+        if type(i) is NewMtl:
+            current_material = i
+        elif type(i) is MapKd:
+            assert current_material is not None
+            yield (current_material, i)
+            current_material = None
+        else:
+            assert False, type(i)
+
+def parse_mtl_file(buf):
+    return list(group_by_material((
+        parsed
+        for line in buf.strip().split('\n')
+        for parsed in parse_mtl_line(line)
+        if not line.startswith('#')
+    )))

parse_obj.py

@@ -0,0 +1,162 @@
+from collections import defaultdict
+from dataclasses import dataclass
+import string
+from fixed_point import FixedPoint
+import fixed_point
+
+@dataclass
+class VertexPosition:
+    x: FixedPoint
+    y: FixedPoint
+    z: FixedPoint
+
+@dataclass
+class VertexNormal:
+    x: FixedPoint
+    y: FixedPoint
+    z: FixedPoint
+
+@dataclass
+class VertexTexture:
+    u: FixedPoint
+    v: FixedPoint
+
+@dataclass
+class IndexVTN:
+    vertex_position: int
+    vertex_texture: int
+    vertex_normal: int
+
+@dataclass
+class Triangle:
+    a: IndexVTN
+    b: IndexVTN
+    c: IndexVTN
+
+@dataclass
+class Quadrilateral:
+    a: IndexVTN
+    b: IndexVTN
+    c: IndexVTN
+    d: IndexVTN
+
+@dataclass
+class Object:
+    name: str
+
+@dataclass
+class Material:
+    lib: str
+    name: str
+
+@dataclass
+class MtlLib:
+    name: str
+
+def parse_fixed_point_vector(args, n):
+    split = args.split()
+    assert len(split) == n, (n, split)
+    return tuple(map(fixed_point.parse, split))
+
+def parse_vertex_position(args):
+    coordinates = parse_fixed_point_vector(args, 3)
+    yield VertexPosition(*coordinates)
+
+def parse_vertex_normal(args):
+    coordinates = parse_fixed_point_vector(args, 3)
+    yield VertexNormal(*coordinates)
+
+def parse_vertex_texture(args):
+    coordinates = parse_fixed_point_vector(args, 2)
+    yield VertexTexture(*coordinates)
+
+def int_minus_one(s):
+    n = int(s) - 1
+    assert n >= 0
+    return n
+
+def _parse_vertex_indices(args):
+    indices = args.split('/')
+    assert len(indices) == 3, indices
+    return IndexVTN(*map(int_minus_one, indices))
+
+def parse_face(args):
+    vertices = args.split()
+    if len(vertices) == 3:
+        yield Triangle(*map(_parse_vertex_indices, vertices))
+    elif len(vertices) == 4:
+        yield Quadrilateral(*map(_parse_vertex_indices, vertices))
+    else:
+        assert False, (len(vertices), args)
+
+def safe(s):
+    return s.replace('-', '_').replace('.', '_').replace(':', '_')
+
+def parse_object(args):
+    name, = args.split()
+    yield Object(safe(name))
+
+def parse_material(args):
+    name, = args.split()
+    yield Material(None, safe(name))
+
+def parse_mtllib(args):
+    name, = args.split()
+    yield MtlLib(name)
+
+def parse_obj_line(line):
+    prefixes = [
+        ('v ', parse_vertex_position),
+        ('vn ', parse_vertex_normal),
+        ('vt ', parse_vertex_texture),
+        ('f ', parse_face),
+        ('o ', parse_object),
+        ('usemtl ', parse_material),
+        ('mtllib ', parse_mtllib),
+    ]
+    for prefix, parser in prefixes:
+        if line.startswith(prefix):
+            args = line.removeprefix(prefix)
+            yield from parser(args)
+
+def group_by_type(l):
+    vertices = defaultdict(list)
+    current_object = None
+    faces = defaultdict(lambda: defaultdict(list))
+    materials = dict()
+    current_mtllib = None
+    multi_material_index = 0
+    for i in l:
+        if type(i) in {VertexPosition, VertexTexture, VertexNormal}:
+            vertices[type(i)].append(i)
+        elif type(i) in {Triangle, Quadrilateral}:
+            assert current_object is not None
+            faces[current_object.name][type(i)].append(i)
+        elif type(i) is Material:
+            assert current_object is not None
+            assert current_mtllib is not None
+            i.lib = current_mtllib.name
+            if current_object.name in materials:
+                if multi_material_index != 0:
+                    current_object.name = current_object.name[:-4]
+                current_object.name += f"_{multi_material_index:03}"
+                multi_material_index += 1
+            assert current_object.name not in materials
+            materials[current_object.name] = i
+        elif type(i) is Object:
+            multi_material_index = 0
+            current_object = i
+        elif type(i) is MtlLib:
+            current_mtllib = i
+        else:
+            assert False, type(i)
+
+    return dict(vertices), dict((k, dict(v)) for k, v in faces.items()), materials
+
+def parse_obj_file(buf):
+    return group_by_type((
+        parsed
+        for line in buf.strip().split('\n')
+        for parsed in parse_obj_line(line)
+        if not line.startswith('#')
+    ))

path.py

@@ -0,0 +1,9 @@
+from os import path
+
+def texture_path(s):
+    #return path.join('..', 'texture', s)
+    return s
+
+def model_path(s):
+    #return path.join('..', 'model', s)
+    return s

profiles.py

@@ -0,0 +1,40 @@
+from dataclasses import dataclass
+
+@dataclass
+class Profile:
+    position: tuple[int, int]
+    texture: tuple[int, int]
+    normal: tuple[int, int]
+
+@dataclass
+class FixedPointBits:
+    integer: int
+    fraction: int
+
+    def to_str(self):
+        return f"{self.integer}.{self.fraction} fixed-point"
+
+@dataclass
+class FloatingPoint:
+    def to_str(self):
+        return f"floating-point"
+
+profiles = {}
+
+profiles["nds"] = Profile(
+    position = FixedPointBits(3, 12), # 3.12
+    normal   = FixedPointBits(0, 9),  # 0.9
+    texture  = FixedPointBits(1, 14), # 1.14
+)
+
+profiles["dreamcast"] = Profile(
+    position = FloatingPoint(),
+    normal = FloatingPoint(),
+    texture = FloatingPoint(),
+)
+
+profiles["saturn"] = Profile(
+    position = FixedPointBits(15, 16),
+    normal = FixedPointBits(15, 16),
+    texture = FixedPointBits(15, 16),
+)

render_material_textures.py

@@ -0,0 +1,119 @@
+from dataclasses import dataclass
+from generate import renderer
+from math import log
+from path import texture_path
+import sys
+
+from PIL import Image
+
+from parse_material import parse_mtl_file
+
+material_filenames = sys.argv[1:]
+
+def render_material_enum(newmtl_mapkd):
+    yield f"enum material {{"
+    for newmtl, mapkd in newmtl_mapkd:
+        yield f"{newmtl.name},";
+    yield "};"
+
+def render_pixel_descriptor(offset, mapkd, dimensions):
+    name, _ext = mapkd.name.rsplit('.', maxsplit=1)
+    pixel_name = f"{name}_data"
+    width, height = dimensions
+    yield ".pixel = {"
+    yield f".start = (uint8_t *)&_binary_{pixel_name}_start,"
+    yield f".size = (int)&_binary_{pixel_name}_size,"
+    yield f".vram_offset = {offset.pixel},"
+    yield f".width = {width},"
+    yield f".height = {height},"
+    yield "},"
+
+def render_palette_descriptor(offset, mapkd, palette_size):
+    name, _ext = mapkd.name.rsplit('.', maxsplit=1)
+    palette_name = f"{name}_data_pal"
+    yield ".palette = {"
+    yield f".start = (uint8_t *)&_binary_{palette_name}_start,"
+    yield f".size = (int)&_binary_{palette_name}_size,"
+    yield f".vram_offset = {offset.palette},"
+    yield f".palette_size = {palette_size},"
+    yield "},"
+
+@dataclass
+class Offset:
+    pixel: int
+    palette: int
+
+def round_up_colors(name, colors):
+    assert colors != 0, (name, colors)
+    if colors <= 4:
+        return 4
+    if colors <= 16:
+        return 16
+    elif colors <= 256:
+        return 256
+    else:
+        assert False, (name, colors)
+
+def image_metadata(mapkd):
+    path = texture_path(mapkd.name)
+    with Image.open(path) as im:
+        dimensions = im.size
+        colors = len(im.palette.colors)
+    return dimensions, colors
+
+def round_up_n(x, multiple):
+    return ((x + multiple - 1) // multiple) * multiple
+
+def bytes_per_pixel(palette_size):
+    bits_per_pixel = int(log(palette_size)/log(2))
+    return bits_per_pixel / 8
+
+def render_material(offset, mapkd):
+    dimensions, colors = image_metadata(mapkd)
+    palette_size = round_up_colors(mapkd.name, colors)
+
+    # pixel descriptor
+    yield from render_pixel_descriptor(offset, mapkd, dimensions)
+    pixel_size = bytes_per_pixel(palette_size) * dimensions[0] * dimensions[1]
+    #pixel_size = 2 * dimensions[0] * dimensions[1]
+    assert int(pixel_size) == pixel_size
+    offset.pixel += round_up_n(int(pixel_size), 8)
+
+    # palette descriptor
+    yield from render_palette_descriptor(offset, mapkd, palette_size)
+    offset.palette += round_up_n(colors * 2, 16)
+
+def render_materials(newmtl_mapkd):
+    yield "struct material_descriptor material[] = {"
+    offset = Offset(0, 0)
+    for newmtl, mapkd in newmtl_mapkd:
+        yield f"[{newmtl.name}] = {{"
+        yield from render_material(offset, mapkd)
+        yield "},"
+    yield "};"
+
+def render_header():
+    yield "#pragma once"
+    yield ""
+    yield "#include <stdint.h>"
+    yield ""
+    yield '#include "model/material.h"'
+    yield ""
+
+if __name__ == "__main__":
+    material_filenames = sys.argv[1:]
+    assert material_filenames
+    newmtl_mapkd = []
+    for material_filename in material_filenames:
+        with open(material_filename) as f:
+            buf = f.read()
+            newmtl_mapkd.extend([
+                (newmtl, mapkd)
+                for (newmtl, mapkd) in parse_mtl_file(buf)
+            ])
+
+    render, out = renderer()
+    render(render_header())
+    render(render_material_enum(newmtl_mapkd))
+    render(render_materials(newmtl_mapkd))
+    sys.stdout.write(out.getvalue())

render_model.py

@@ -0,0 +1,174 @@
+from dataclasses import astuple
+import sys
+from generate import renderer
+import math
+
+import fixed_point
+
+from parse_obj import parse_obj_file
+from parse_obj import VertexPosition
+from parse_obj import VertexNormal
+from parse_obj import VertexTexture
+from parse_obj import Triangle
+from parse_obj import Quadrilateral
+
+import profiles
+
+def render_index_vtn(index_vtn):
+    s = ", ".join(map(str, index_vtn))
+    yield f"{{{s}}},"
+
+def render_face(face):
+    yield "{ .v = {"
+    for index_vtn in astuple(face):
+        yield from render_index_vtn(index_vtn)
+    yield "}},"
+
+def render_faces(prefix, name, faces):
+    yield f"union {name} {prefix}_{name}[] = {{"
+    for face in faces:
+        yield from render_face(face)
+    yield "};"
+
+def render_triangles(prefix, faces):
+    yield from render_faces(prefix, "triangle", faces)
+
+def render_quadrilateral(prefix, faces):
+    yield from render_faces(prefix, "quadrilateral", faces)
+
+def unit_vector(vec):
+    x = vec.x.to_float()
+    y = vec.y.to_float()
+    z = vec.z.to_float()
+    norm = math.sqrt(x ** 2 + y ** 2 + z ** 2)
+    return type(vec)(
+        fixed_point.parse(str(x / norm)),
+        fixed_point.parse(str(y / norm)),
+        fixed_point.parse(str(z / norm))
+    )
+
+def xyz(vec):
+    return (vec.x, vec.y, vec.z)
+
+def uv(vec):
+    return (vec.u, vec.v)
+
+def render_vertex(profile_item, vertex_tuple):
+    def _profile_item(profile_item, fp):
+        if type(profile_item) == profiles.FixedPointBits:
+            return fp.to_fixed_point(profile_item.integer, profile_item.fraction).to_int()
+        elif type(profile_item) == profiles.FloatingPoint:
+            return fp.to_float()
+        else:
+            assert False, type(profile_item)
+
+    s = ", ".join(
+        str(_profile_item(profile_item, fp))
+        for fp in vertex_tuple
+    )
+    yield f"{{{s}}},"
+
+def render_vertices(profile_item, prefix, name, vertices):
+    yield f"// {profile_item.to_str()}"
+    yield f"vertex_{name} {prefix}_{name}[] = {{"
+    for i, vertex in enumerate(vertices):
+        yield from render_vertex(profile_item, vertex)
+    yield "};"
+
+def render_vertex_positions(profile, prefix, vertex_positions):
+    yield from render_vertices(profile.position,
+                               prefix,
+                               "position",
+                               map(xyz, vertex_positions))
+
+def render_vertex_normals(profile, prefix, vertex_normals):
+    yield from render_vertices(profile.normal,
+                               prefix,
+                               "normal",
+                               map(xyz, map(unit_vector, vertex_normals)))
+
+def render_vertex_texture(profile, prefix, vertex_textures):
+    yield from render_vertices(profile.texture,
+                               prefix,
+                               "texture",
+                               map(uv, vertex_textures))
+
+def render_object(prefix, object_name, d, material):
+    yield f"struct object {prefix}_{object_name} = {{"
+
+    triangle_count = len(d[Triangle]) if Triangle in d else 0
+    quadrilateral_count = len(d[Quadrilateral]) if Quadrilateral in d else 0
+
+    if triangle_count > 0:
+        yield f".triangle = &{prefix}_{object_name}_triangle[0],"
+    else:
+        yield f".triangle = NULL,"
+
+    if quadrilateral_count > 0:
+        yield f".quadrilateral = &{prefix}_{object_name}_quadrilateral[0],"
+    else:
+        yield f".quadrilateral = NULL,"
+
+    yield f".triangle_count = {triangle_count},"
+    yield f".quadrilateral_count = {quadrilateral_count},"
+
+    if material is None:
+        yield f".material = -1,",
+    else:
+        yield f".material = {material.name},"
+
+    yield "};"
+
+def render_object_list(prefix, object_names):
+    yield f"struct object * {prefix}_object_list[] = {{"
+    for object_name in object_names:
+        yield f"&{prefix}_{object_name},"
+    yield "};"
+
+def render_model(prefix, object_count):
+    yield f"struct model {prefix}_model = {{"
+    yield f".position = &{prefix}_position[0],"
+    yield f".texture = &{prefix}_texture[0],"
+    yield f".normal = &{prefix}_normal[0],"
+    yield f".object = &{prefix}_object_list[0],"
+    yield f".object_count = {object_count},"
+    yield "};"
+
+def render_header():
+    yield "#pragma once"
+    yield ""
+    yield "#include <stddef.h>"
+    yield ""
+    yield '#include "../model.h"'
+    yield ""
+
+obj_filename = sys.argv[1]
+prefix = sys.argv[2]
+profile_name = sys.argv[3]
+
+profile = profiles.profiles[profile_name]
+
+with open(obj_filename) as f:
+    buf = f.read()
+
+vertices, faces, materials = parse_obj_file(buf)
+render, out = renderer()
+render(render_header())
+render(render_vertex_positions(profile, prefix, vertices[VertexPosition]))
+render(render_vertex_texture(profile, prefix, vertices[VertexTexture]))
+render(render_vertex_normals(profile, prefix, vertices[VertexNormal]))
+
+for object_name, d in faces.items():
+    object_prefix = '_'.join((prefix, object_name))
+
+    if Triangle in d:
+        render(render_triangles(object_prefix, d[Triangle]))
+    if Quadrilateral in d:
+        render(render_quadrilateral(object_prefix, d[Quadrilateral]))
+
+    render(render_object(prefix, object_name, d, materials.get(object_name)));
+
+render(render_object_list(prefix, faces.keys()))
+render(render_model(prefix, len(faces)))
+
+sys.stdout.write(out.getvalue())