From 25156e2fc0acc0bf73ea99bc50f2a4a4cba0aa26 Mon Sep 17 00:00:00 2001 From: Zack Buhman Date: Fri, 13 Sep 2024 05:58:57 -0500 Subject: [PATCH] initial --- binary_image_palette.py | 78 ++++++++++++++++ fixed_point.py | 77 ++++++++++++++++ generate.py | 35 ++++++++ parse_material.py | 47 ++++++++++ parse_obj.py | 162 +++++++++++++++++++++++++++++++++ path.py | 9 ++ profiles.py | 40 +++++++++ render_material_textures.py | 119 ++++++++++++++++++++++++ render_model.py | 174 ++++++++++++++++++++++++++++++++++++ 9 files changed, 741 insertions(+) create mode 100644 binary_image_palette.py create mode 100644 fixed_point.py create mode 100644 generate.py create mode 100644 parse_material.py create mode 100644 parse_obj.py create mode 100644 path.py create mode 100644 profiles.py create mode 100644 render_material_textures.py create mode 100644 render_model.py diff --git a/binary_image_palette.py b/binary_image_palette.py new file mode 100644 index 0000000..bc967e7 --- /dev/null +++ b/binary_image_palette.py @@ -0,0 +1,78 @@ +import struct +import sys + +from PIL import Image + +def round_up_palette_size(palette_size): + assert palette_size != 0, (name, palette_size) + if palette_size <= 4: + return 4 + elif palette_size <= 16: + return 16 + elif palette_size <= 256: + return 256 + else: + assert False, palette_size + +def pixels_per_byte(palette_size): + if palette_size == 4: + return 4 + elif palette_size == 16: + return 2 + elif palette_size == 256: + return 1 + else: + assert False, palette_size + +def pack_one_byte(pixels, index, colors, palette_size): + color_count = len(colors) + num = pixels_per_byte(palette_size) + shift = 8 // num + byte = 0 + i = 0 + while num > 0: + px, alpha = pixels[index] + if alpha == 0 and color_count < palette_size: + px = color_count + assert px < palette_size + byte |= px << (shift * i) + index += 1 + i += 1 + num -= 1 + return [byte], index + +def pack_one_texel(pixels, index, colors, palette_size): + px, alpha = pixels[index] + return + +def pack_pixels(pixels, width, height, colors, palette_size): + index = 0 + with open(sys.argv[2], 'wb') as f: + while index < (width * height): + byte_list, index = pack_texel(pixels, index, colors, palette_size) + f.write(bytes(byte)) + +def pack_palette(colors, palette_size): + with open(sys.argv[2], 'wb') as f: + for color in colors: + out = argb1555(255, *color) + f.write(struct.pack('= integer_point: + raise FixedPointOverflow((integer, integer_point)) + return FixedPoint( + fp.sign, + value, + point + ) + + def to_int(fp): + return fp.sign * fp.value + + def to_float(fp): + return fp.sign * fp.value / fp.point + +def from_float(n): + if n == 0.0: + sign = 1 + value = 0 + else: + sign = -1 if n < 0 else 1 + value = abs(round(n * (2 ** 32))) + point = 2 ** 32 + return FixedPoint(sign, value, point) + +assert from_float(0.5).to_float() == 0.5 +assert from_float(1.5).to_fixed_point(16, 16).value == 98304 + +def parse(s): + sign = -1 if s.startswith('-') else 1 + s = s.removeprefix('-') + integer, fraction = s.split('.') + assert all(c in string.digits for c in integer), integer + assert all(c in string.digits for c in fraction), fraction + assert len(integer) > 0 and len(fraction) > 0, s + point = 10 ** len(fraction) + value = int(integer) * point + int(fraction) + return FixedPoint( + sign, + value, + point + ) + +def equal(a, b): + epsilon = 0.00001 + return (a - b) < epsilon + +def assert_raises(e, f): + try: + f() + except e: + return + raise AssertionError(f"expected {str(e)}") + +assert parse("1.234").value == 1234 +assert equal(parse("1.234").to_float(), 1.234) +assert parse("1.234").to_fixed_point(16, 16).value == 80871 +assert_raises(FixedPointOverflow, + lambda: parse("2.234").to_fixed_point(1, 16)) +assert parse("2.234").to_fixed_point(2, 16).value == 146407 diff --git a/generate.py b/generate.py new file mode 100644 index 0000000..2871189 --- /dev/null +++ b/generate.py @@ -0,0 +1,35 @@ +import io + +def should_autonewline(line): + return ( + "static_assert" not in line + and "extern" not in line + and (len(line.split()) < 2 or line.split()[1] != '=') # hacky; meh + ) + +def _render(out, lines): + indent = " " + level = 0 + for l in lines: + if l and (l[0] == "}" or l[0] == ")"): + level -= 2 + assert level >= 0, out.getvalue() + + if len(l) == 0: + out.write("\n") + else: + out.write(indent * level + l + "\n") + + if l and (l[-1] == "{" or l[-1] == "("): + level += 2 + + if level == 0 and l and l[-1] == ";": + if should_autonewline(l): + out.write("\n") + return out + +def renderer(): + out = io.StringIO() + def render(lines): + return _render(out, lines) + return render, out diff --git a/parse_material.py b/parse_material.py new file mode 100644 index 0000000..04d0793 --- /dev/null +++ b/parse_material.py @@ -0,0 +1,47 @@ +from dataclasses import dataclass + +@dataclass +class NewMtl: + name: str + +@dataclass +class MapKd: + name: str + +def parse_material_newmtl(args): + name, = args.split() + yield NewMtl(name.replace('-', '_').replace('.', '_')) + +def parse_material_mapkd(args): + name, = args.split() + yield MapKd(name) + +def parse_mtl_line(line): + prefixes = [ + ('newmtl ', parse_material_newmtl), + ('map_Kd ', parse_material_mapkd), + ] + for prefix, parser in prefixes: + if line.startswith(prefix): + args = line.removeprefix(prefix) + yield from parser(args) + +def group_by_material(l): + current_material = None + for i in l: + if type(i) is NewMtl: + current_material = i + elif type(i) is MapKd: + assert current_material is not None + yield (current_material, i) + current_material = None + else: + assert False, type(i) + +def parse_mtl_file(buf): + return list(group_by_material(( + parsed + for line in buf.strip().split('\n') + for parsed in parse_mtl_line(line) + if not line.startswith('#') + ))) diff --git a/parse_obj.py b/parse_obj.py new file mode 100644 index 0000000..f004ffc --- /dev/null +++ b/parse_obj.py @@ -0,0 +1,162 @@ +from collections import defaultdict +from dataclasses import dataclass +import string +from fixed_point import FixedPoint +import fixed_point + +@dataclass +class VertexPosition: + x: FixedPoint + y: FixedPoint + z: FixedPoint + +@dataclass +class VertexNormal: + x: FixedPoint + y: FixedPoint + z: FixedPoint + +@dataclass +class VertexTexture: + u: FixedPoint + v: FixedPoint + +@dataclass +class IndexVTN: + vertex_position: int + vertex_texture: int + vertex_normal: int + +@dataclass +class Triangle: + a: IndexVTN + b: IndexVTN + c: IndexVTN + +@dataclass +class Quadrilateral: + a: IndexVTN + b: IndexVTN + c: IndexVTN + d: IndexVTN + +@dataclass +class Object: + name: str + +@dataclass +class Material: + lib: str + name: str + +@dataclass +class MtlLib: + name: str + +def parse_fixed_point_vector(args, n): + split = args.split() + assert len(split) == n, (n, split) + return tuple(map(fixed_point.parse, split)) + +def parse_vertex_position(args): + coordinates = parse_fixed_point_vector(args, 3) + yield VertexPosition(*coordinates) + +def parse_vertex_normal(args): + coordinates = parse_fixed_point_vector(args, 3) + yield VertexNormal(*coordinates) + +def parse_vertex_texture(args): + coordinates = parse_fixed_point_vector(args, 2) + yield VertexTexture(*coordinates) + +def int_minus_one(s): + n = int(s) - 1 + assert n >= 0 + return n + +def _parse_vertex_indices(args): + indices = args.split('/') + assert len(indices) == 3, indices + return IndexVTN(*map(int_minus_one, indices)) + +def parse_face(args): + vertices = args.split() + if len(vertices) == 3: + yield Triangle(*map(_parse_vertex_indices, vertices)) + elif len(vertices) == 4: + yield Quadrilateral(*map(_parse_vertex_indices, vertices)) + else: + assert False, (len(vertices), args) + +def safe(s): + return s.replace('-', '_').replace('.', '_').replace(':', '_') + +def parse_object(args): + name, = args.split() + yield Object(safe(name)) + +def parse_material(args): + name, = args.split() + yield Material(None, safe(name)) + +def parse_mtllib(args): + name, = args.split() + yield MtlLib(name) + +def parse_obj_line(line): + prefixes = [ + ('v ', parse_vertex_position), + ('vn ', parse_vertex_normal), + ('vt ', parse_vertex_texture), + ('f ', parse_face), + ('o ', parse_object), + ('usemtl ', parse_material), + ('mtllib ', parse_mtllib), + ] + for prefix, parser in prefixes: + if line.startswith(prefix): + args = line.removeprefix(prefix) + yield from parser(args) + +def group_by_type(l): + vertices = defaultdict(list) + current_object = None + faces = defaultdict(lambda: defaultdict(list)) + materials = dict() + current_mtllib = None + multi_material_index = 0 + for i in l: + if type(i) in {VertexPosition, VertexTexture, VertexNormal}: + vertices[type(i)].append(i) + elif type(i) in {Triangle, Quadrilateral}: + assert current_object is not None + faces[current_object.name][type(i)].append(i) + elif type(i) is Material: + assert current_object is not None + assert current_mtllib is not None + i.lib = current_mtllib.name + if current_object.name in materials: + if multi_material_index != 0: + current_object.name = current_object.name[:-4] + current_object.name += f"_{multi_material_index:03}" + multi_material_index += 1 + assert current_object.name not in materials + materials[current_object.name] = i + elif type(i) is Object: + multi_material_index = 0 + current_object = i + elif type(i) is MtlLib: + current_mtllib = i + else: + assert False, type(i) + + return dict(vertices), dict((k, dict(v)) for k, v in faces.items()), materials + +def parse_obj_file(buf): + return group_by_type(( + parsed + for line in buf.strip().split('\n') + for parsed in parse_obj_line(line) + if not line.startswith('#') + )) diff --git a/path.py b/path.py new file mode 100644 index 0000000..2c3511a --- /dev/null +++ b/path.py @@ -0,0 +1,9 @@ +from os import path + +def texture_path(s): + #return path.join('..', 'texture', s) + return s + +def model_path(s): + #return path.join('..', 'model', s) + return s diff --git a/profiles.py b/profiles.py new file mode 100644 index 0000000..202bd83 --- /dev/null +++ b/profiles.py @@ -0,0 +1,40 @@ +from dataclasses import dataclass + +@dataclass +class Profile: + position: tuple[int, int] + texture: tuple[int, int] + normal: tuple[int, int] + +@dataclass +class FixedPointBits: + integer: int + fraction: int + + def to_str(self): + return f"{self.integer}.{self.fraction} fixed-point" + +@dataclass +class FloatingPoint: + def to_str(self): + return f"floating-point" + +profiles = {} + +profiles["nds"] = Profile( + position = FixedPointBits(3, 12), # 3.12 + normal = FixedPointBits(0, 9), # 0.9 + texture = FixedPointBits(1, 14), # 1.14 +) + +profiles["dreamcast"] = Profile( + position = FloatingPoint(), + normal = FloatingPoint(), + texture = FloatingPoint(), +) + +profiles["saturn"] = Profile( + position = FixedPointBits(15, 16), + normal = FixedPointBits(15, 16), + texture = FixedPointBits(15, 16), +) diff --git a/render_material_textures.py b/render_material_textures.py new file mode 100644 index 0000000..f7ae804 --- /dev/null +++ b/render_material_textures.py @@ -0,0 +1,119 @@ +from dataclasses import dataclass +from generate import renderer +from math import log +from path import texture_path +import sys + +from PIL import Image + +from parse_material import parse_mtl_file + +material_filenames = sys.argv[1:] + +def render_material_enum(newmtl_mapkd): + yield f"enum material {{" + for newmtl, mapkd in newmtl_mapkd: + yield f"{newmtl.name},"; + yield "};" + +def render_pixel_descriptor(offset, mapkd, dimensions): + name, _ext = mapkd.name.rsplit('.', maxsplit=1) + pixel_name = f"{name}_data" + width, height = dimensions + yield ".pixel = {" + yield f".start = (uint8_t *)&_binary_{pixel_name}_start," + yield f".size = (int)&_binary_{pixel_name}_size," + yield f".vram_offset = {offset.pixel}," + yield f".width = {width}," + yield f".height = {height}," + yield "}," + +def render_palette_descriptor(offset, mapkd, palette_size): + name, _ext = mapkd.name.rsplit('.', maxsplit=1) + palette_name = f"{name}_data_pal" + yield ".palette = {" + yield f".start = (uint8_t *)&_binary_{palette_name}_start," + yield f".size = (int)&_binary_{palette_name}_size," + yield f".vram_offset = {offset.palette}," + yield f".palette_size = {palette_size}," + yield "}," + +@dataclass +class Offset: + pixel: int + palette: int + +def round_up_colors(name, colors): + assert colors != 0, (name, colors) + if colors <= 4: + return 4 + if colors <= 16: + return 16 + elif colors <= 256: + return 256 + else: + assert False, (name, colors) + +def image_metadata(mapkd): + path = texture_path(mapkd.name) + with Image.open(path) as im: + dimensions = im.size + colors = len(im.palette.colors) + return dimensions, colors + +def round_up_n(x, multiple): + return ((x + multiple - 1) // multiple) * multiple + +def bytes_per_pixel(palette_size): + bits_per_pixel = int(log(palette_size)/log(2)) + return bits_per_pixel / 8 + +def render_material(offset, mapkd): + dimensions, colors = image_metadata(mapkd) + palette_size = round_up_colors(mapkd.name, colors) + + # pixel descriptor + yield from render_pixel_descriptor(offset, mapkd, dimensions) + pixel_size = bytes_per_pixel(palette_size) * dimensions[0] * dimensions[1] + #pixel_size = 2 * dimensions[0] * dimensions[1] + assert int(pixel_size) == pixel_size + offset.pixel += round_up_n(int(pixel_size), 8) + + # palette descriptor + yield from render_palette_descriptor(offset, mapkd, palette_size) + offset.palette += round_up_n(colors * 2, 16) + +def render_materials(newmtl_mapkd): + yield "struct material_descriptor material[] = {" + offset = Offset(0, 0) + for newmtl, mapkd in newmtl_mapkd: + yield f"[{newmtl.name}] = {{" + yield from render_material(offset, mapkd) + yield "}," + yield "};" + +def render_header(): + yield "#pragma once" + yield "" + yield "#include " + yield "" + yield '#include "model/material.h"' + yield "" + +if __name__ == "__main__": + material_filenames = sys.argv[1:] + assert material_filenames + newmtl_mapkd = [] + for material_filename in material_filenames: + with open(material_filename) as f: + buf = f.read() + newmtl_mapkd.extend([ + (newmtl, mapkd) + for (newmtl, mapkd) in parse_mtl_file(buf) + ]) + + render, out = renderer() + render(render_header()) + render(render_material_enum(newmtl_mapkd)) + render(render_materials(newmtl_mapkd)) + sys.stdout.write(out.getvalue()) diff --git a/render_model.py b/render_model.py new file mode 100644 index 0000000..6ac1851 --- /dev/null +++ b/render_model.py @@ -0,0 +1,174 @@ +from dataclasses import astuple +import sys +from generate import renderer +import math + +import fixed_point + +from parse_obj import parse_obj_file +from parse_obj import VertexPosition +from parse_obj import VertexNormal +from parse_obj import VertexTexture +from parse_obj import Triangle +from parse_obj import Quadrilateral + +import profiles + +def render_index_vtn(index_vtn): + s = ", ".join(map(str, index_vtn)) + yield f"{{{s}}}," + +def render_face(face): + yield "{ .v = {" + for index_vtn in astuple(face): + yield from render_index_vtn(index_vtn) + yield "}}," + +def render_faces(prefix, name, faces): + yield f"union {name} {prefix}_{name}[] = {{" + for face in faces: + yield from render_face(face) + yield "};" + +def render_triangles(prefix, faces): + yield from render_faces(prefix, "triangle", faces) + +def render_quadrilateral(prefix, faces): + yield from render_faces(prefix, "quadrilateral", faces) + +def unit_vector(vec): + x = vec.x.to_float() + y = vec.y.to_float() + z = vec.z.to_float() + norm = math.sqrt(x ** 2 + y ** 2 + z ** 2) + return type(vec)( + fixed_point.parse(str(x / norm)), + fixed_point.parse(str(y / norm)), + fixed_point.parse(str(z / norm)) + ) + +def xyz(vec): + return (vec.x, vec.y, vec.z) + +def uv(vec): + return (vec.u, vec.v) + +def render_vertex(profile_item, vertex_tuple): + def _profile_item(profile_item, fp): + if type(profile_item) == profiles.FixedPointBits: + return fp.to_fixed_point(profile_item.integer, profile_item.fraction).to_int() + elif type(profile_item) == profiles.FloatingPoint: + return fp.to_float() + else: + assert False, type(profile_item) + + s = ", ".join( + str(_profile_item(profile_item, fp)) + for fp in vertex_tuple + ) + yield f"{{{s}}}," + +def render_vertices(profile_item, prefix, name, vertices): + yield f"// {profile_item.to_str()}" + yield f"vertex_{name} {prefix}_{name}[] = {{" + for i, vertex in enumerate(vertices): + yield from render_vertex(profile_item, vertex) + yield "};" + +def render_vertex_positions(profile, prefix, vertex_positions): + yield from render_vertices(profile.position, + prefix, + "position", + map(xyz, vertex_positions)) + +def render_vertex_normals(profile, prefix, vertex_normals): + yield from render_vertices(profile.normal, + prefix, + "normal", + map(xyz, map(unit_vector, vertex_normals))) + +def render_vertex_texture(profile, prefix, vertex_textures): + yield from render_vertices(profile.texture, + prefix, + "texture", + map(uv, vertex_textures)) + +def render_object(prefix, object_name, d, material): + yield f"struct object {prefix}_{object_name} = {{" + + triangle_count = len(d[Triangle]) if Triangle in d else 0 + quadrilateral_count = len(d[Quadrilateral]) if Quadrilateral in d else 0 + + if triangle_count > 0: + yield f".triangle = &{prefix}_{object_name}_triangle[0]," + else: + yield f".triangle = NULL," + + if quadrilateral_count > 0: + yield f".quadrilateral = &{prefix}_{object_name}_quadrilateral[0]," + else: + yield f".quadrilateral = NULL," + + yield f".triangle_count = {triangle_count}," + yield f".quadrilateral_count = {quadrilateral_count}," + + if material is None: + yield f".material = -1,", + else: + yield f".material = {material.name}," + + yield "};" + +def render_object_list(prefix, object_names): + yield f"struct object * {prefix}_object_list[] = {{" + for object_name in object_names: + yield f"&{prefix}_{object_name}," + yield "};" + +def render_model(prefix, object_count): + yield f"struct model {prefix}_model = {{" + yield f".position = &{prefix}_position[0]," + yield f".texture = &{prefix}_texture[0]," + yield f".normal = &{prefix}_normal[0]," + yield f".object = &{prefix}_object_list[0]," + yield f".object_count = {object_count}," + yield "};" + +def render_header(): + yield "#pragma once" + yield "" + yield "#include " + yield "" + yield '#include "../model.h"' + yield "" + +obj_filename = sys.argv[1] +prefix = sys.argv[2] +profile_name = sys.argv[3] + +profile = profiles.profiles[profile_name] + +with open(obj_filename) as f: + buf = f.read() + +vertices, faces, materials = parse_obj_file(buf) +render, out = renderer() +render(render_header()) +render(render_vertex_positions(profile, prefix, vertices[VertexPosition])) +render(render_vertex_texture(profile, prefix, vertices[VertexTexture])) +render(render_vertex_normals(profile, prefix, vertices[VertexNormal])) + +for object_name, d in faces.items(): + object_prefix = '_'.join((prefix, object_name)) + + if Triangle in d: + render(render_triangles(object_prefix, d[Triangle])) + if Quadrilateral in d: + render(render_quadrilateral(object_prefix, d[Quadrilateral])) + + render(render_object(prefix, object_name, d, materials.get(object_name))); + +render(render_object_list(prefix, faces.keys())) +render(render_model(prefix, len(faces))) + +sys.stdout.write(out.getvalue())