minecraft: instanced rendering grouped by cube configuration

Makefile

@@ -2,7 +2,7 @@
 CC=$(PREFIX)gcc
 CXX=$(PREFIX)g++
 
-OPT = -Og -march=x86-64-v3
+OPT = -O0 -march=x86-64-v3
 
 CSTD = -std=gnu23
 CXXSTD = -std=gnu++23

include/data.inc

@@ -0,0 +1,10 @@
+short index_buffer_configuration_offsets[] = {
+    0,    0,    6,   12,   24,   30,   42,   54,
+   72,   78,   90,  102,  120,  132,  150,  168,
+  192,  198,  210,  222,  240,  252,  270,  288,
+  312,  324,  342,  360,  384,  402,  426,  450,
+  480,  486,  498,  510,  528,  540,  558,  576,
+  600,  612,  630,  648,  672,  690,  714,  738,
+  768,  780,  798,  816,  840,  858,  882,  906,
+  936,  954,  978, 1002, 1032, 1056, 1086, 1116,
+};

minecraft/gen/mc.py

@@ -2,6 +2,7 @@ import sys
 import struct
 from pprint import pprint
 from itertools import chain
+from collections import defaultdict
 
 import mcregion
 import vec3
@@ -16,15 +17,6 @@ def wrap_n(nc, chunk_c):
         chunk_c = chunk_c + 1
     return nc, chunk_c
 
-normals = [
-    (-1.0, 0.0, 0.0),
-    (0.0, -1.0, 0.0),
-    (0.0, 0.0, -1.0),
-    (0.0, 0.0, 1.0),
-    (0.0, 1.0, 0.0),
-    (1.0, 0.0, 0.0),
-]
-
 def block_neighbors(level_table, chunk_x, chunk_z, block_index):
     block_id = level_table[(chunk_x, chunk_z)].blocks[block_index]
     if block_id == 0:
@@ -51,7 +43,7 @@ def block_neighbors(level_table, chunk_x, chunk_z, block_index):
     center_position = vec3.add((x, y, z), (chunk_x * 16, 0, chunk_z * 16))
 
     def find_non_neighbors():
-        for i, normal in enumerate(normals):
+        for i, normal in enumerate(vertex_buffer.normals):
             neighbor = vec3.add(normal, (x, y, z))
             if not neighbor_exists(*neighbor):
                 yield i
@@ -87,17 +79,38 @@ def build_block_configuration_table():
         indices = []
         for j in range(6):
             if ((i >> j) & 1) != 0:
-                indices.extend(vertex_buffer.faces_by_normal[normals[j]])
+                indices.extend(vertex_buffer.faces_by_normal[vertex_buffer.normals[j]])
         yield indices
 
-def build_block_instances(f, blocks):
+def build_block_instances(blocks):
+    by_configuration = defaultdict(list)
+
     for position, block_id, normal_indices in blocks:
-        block_configuration = normal_indices_as_block_configuration(normal_indices)
+        configuration = normal_indices_as_block_configuration(normal_indices)
         #print(position, block_id, block_configuration)
-        f.write(struct.pack("<hhhBB",
+        by_configuration[configuration].append((position, block_id))
+
+    offset = 0
+    configuration_instance_count_offset = []
+    with open(f"{data_path}.instance.vtx", "wb") as f:
+        for configuration in range(64):
+            if configuration not in by_configuration:
+                configuration_instance_count_offset.append((0, 0))
+                continue
+            blocks = by_configuration[configuration]
+            configuration_instance_count_offset.append((len(blocks), offset))
+            for position, block_id in blocks:
+                packed = struct.pack("<hhhBB",
                                      position[0], position[1], position[2],
                                      block_id,
-                            block_configuration))
+                                     0)
+                f.write(packed)
+                offset += len(packed)
+
+    with open(f"{data_path}.instance.cfg", "wb") as f:
+        for instance_count, offset in configuration_instance_count_offset:
+            print(instance_count, offset)
+            f.write(struct.pack("<ii", instance_count, offset))
 
 def main(mcr_path, data_path):
     with open(mcr_path, "rb") as f:
@@ -111,9 +124,7 @@ def main(mcr_path, data_path):
 
     level_table = build_level_table(mem, locations)
     blocks = devoxelize_region(level_table)
-
+    build_block_instances(blocks)
-    with open(data_path + ".vtx", "wb") as f:
-        build_block_instances(f, blocks)
 
     #pprint(list(build_block_configuration_table()))
 

minecraft/gen/vertex_buffer.py

@@ -1,3 +1,4 @@
+import struct
 import vec3
 
 vertex_table = [
@@ -36,29 +37,42 @@ faces_by_normal = {
     (1.0, 0.0, 0.0): [12, 13, 14, 12, 22, 13]
 }
 
-vertex_buffer = {}
+normals = [
+    (-1.0, 0.0, 0.0),
+    (0.0, -1.0, 0.0),
+    (0.0, 0.0, -1.0),
+    (0.0, 0.0, 1.0),
+    (0.0, 1.0, 0.0),
+    (1.0, 0.0, 0.0),
+]
 
-def add_vertex(vertex):
+def build_configuration_index_buffers(f):
-    if vertex in vertex_buffer:
+    offset = 0
-        return vertex_buffer[vertex]
+    configuration_offsets = []
-    else:
+    for configuration in range(64):
-        index = len(vertex_buffer)
+        configuration_offsets.append(offset)
-        vertex_buffer[vertex] = index
+        for i in range(6):
-        return index
+            if (configuration & (1 << i)) == 0:
+                continue
+            normal = normals[i]
+            indices = faces_by_normal[normal]
+            for index in indices:
+                f.write(struct.pack("<B", index))
+                offset += 1
 
-def emit_face(center_position, block_id, triangles):
+    for i, offset in enumerate(configuration_offsets):
-    for index in triangles:
+        print(str(offset).rjust(4), end=", ")
-        position, normal, texture = vertex_table[index]
+        if i % 8 == 7:
-        position = vec3.add(vec3.mul(position, 0.5), center_position)
+            print()
-        vertex = (position, normal, texture, block_id)
-        new_index = add_vertex(vertex)
-        yield new_index
 
-def linearized_vertex_buffer():
+def build_vertex_buffer(f):
-    for vertex, i in sorted(vertex_buffer.items(), key=lambda kv: kv[1]):
+    for position, normal, texture in vertex_table:
-        yield vertex
+        position = vec3.mul(position, 0.5)
+        f.write(struct.pack("<eeeeeeee", *position, *normal, *texture))
 
-#with open(data_path + ".vtx", "wb") as f:
+if __name__ == "__main__":
-#    for vertex in linearized_vertex_buffer():
+    with open("configuration.idx", "wb") as f:
-#        vertex = [*vertex[0], *vertex[1], *vertex[2], vertex[3]]#, vertex[3]]
+        build_configuration_index_buffers(f)
-#        f.write(struct.pack("<fffffffff", *vertex))
+
+    with open("per_vertex.vtx", "wb") as f:
+        build_vertex_buffer(f)

shader/test.frag

@@ -1,10 +1,10 @@
 #version 330 core
 
-in GS_OUT {
+in VS_OUT {
   vec3 Normal;
   vec2 Texture;
   flat int BlockID;
-} gs_out;
+} fs_in;
 
 out vec4 FragColor;
 
@@ -32,13 +32,13 @@ int Textures[256] = int[256](
 void main()
 {
   vec3 light_direction = normalize(vec3(-1, -0.5, 0.5));
-  float diffuse_intensity = max(dot(normalize(gs_out.Normal), light_direction), 0.0);
+  float diffuse_intensity = max(dot(normalize(fs_in.Normal), light_direction), 0.0);
 
-  int terrain_ix = int(Textures[int(gs_out.BlockID)]);
+  int terrain_ix = int(Textures[int(fs_in.BlockID)]);
   int terrain_x = terrain_ix % 16;
   int terrain_y = terrain_ix / 16;
   ivec2 coord = ivec2(terrain_x, terrain_y) * 16;
-  coord += ivec2(gs_out.Texture.xy * 16.0);
+  coord += ivec2(fs_in.Texture.xy * 16.0);
 
   vec4 texture_color = texelFetch(TerrainSampler, coord, 0);
   if (texture_color.w != 1.0) {
@@ -46,8 +46,11 @@ void main()
     return;
   }
 
-  if (int(gs_out.BlockID) == 18) // leaves
+  if (int(fs_in.BlockID) == 18) // leaves
     texture_color.xyz *= vec3(0.125, 0.494, 0.027);
 
+  if (diffuse_intensity < 0.1)
+    diffuse_intensity = 0.1;
+
   FragColor = vec4(texture_color.xyz * vec3(diffuse_intensity), 1.0);
 }

shader/test.geom

@@ -63,25 +63,22 @@ vec4 transform(vec3 p)
 void emit_face(vec3 normal, int face[4])
 {
   vec3 position = gl_in[0].gl_Position.xyz;
+  gs_out.Normal = normal.xzy;
 
   PT vtx0 = vertices[face[0]];
   gl_Position = transform(position + vtx0.Position * 0.5);
-  gs_out.Normal = normal.xzy;
   gs_out.Texture = vtx0.Texture;
   EmitVertex();
   PT vtx1 = vertices[face[1]];
   gl_Position = transform(position + vtx1.Position * 0.5);
-  gs_out.Normal = normal.xzy;
   gs_out.Texture = vtx1.Texture;
   EmitVertex();
   PT vtx2 = vertices[face[2]];
   gl_Position = transform(position + vtx2.Position * 0.5);
-  gs_out.Normal = normal.xzy;
   gs_out.Texture = vtx2.Texture;
   EmitVertex();
   PT vtx3 = vertices[face[3]];
   gl_Position = transform(position + vtx3.Position * 0.5);
-  gs_out.Normal = normal.xzy;
   gs_out.Texture = vtx3.Texture;
   EmitVertex();
   EndPrimitive();

shader/test.vert

@@ -1,20 +1,26 @@
 #version 330 core
 
-layout (location = 0) in vec3 Position;
+// per-vertex:
-layout (location = 1) in float BlockID;
+in vec3 Position;
-layout (location = 2) in float Configuration;
+in vec3 Normal;
+in vec2 Texture;
+// per-instance:
+in vec3 BlockPosition;
+in float BlockID;
 
 out VS_OUT {
-  int BlockID;
+  vec3 Normal;
-  int Configuration;
+  vec2 Texture;
+  flat int BlockID;
 } vs_out;
 
 uniform mat4 Transform;
 
 void main()
 {
+  vs_out.Normal = Normal.xzy;
+  vs_out.Texture = Texture;
   vs_out.BlockID = int(BlockID);
-  vs_out.Configuration = int(Configuration);
 
-  gl_Position = vec4(Position.xyz, 1.0);
+  gl_Position = Transform * vec4((Position + BlockPosition).xzy, 1.0);
 }

src/test.cpp

@@ -1,16 +1,22 @@
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 
 #include "glad/gl.h"
 #include "opengl.h"
 #include "directxmath/directxmath.h"
 #include "test.h"
 
+#include "data.inc"
+
 struct location {
   struct {
     unsigned int position;
+    unsigned int normal;
+    unsigned int texture;
+    unsigned int block_position;
     unsigned int block_id;
-    unsigned int configuration;
+    //unsigned int configuration;
   } attrib;
   struct {
     unsigned int transform;
@@ -22,27 +28,55 @@ struct location {
 static unsigned int test_program;
 static struct location location;
 
-static unsigned int vertex_array_objects[4];
+struct char_tpl {
-static unsigned int vertex_buffers[4];
+  const char * vtx;
-static unsigned int vertex_count[4];
+  const char * cfg;
-//static unsigned int index_buffers[4];
+};
-//static unsigned int index_count[4];
+
+static const int region_count = 4;
+static const char_tpl vertex_paths[region_count] = {
+  { "minecraft/region.0.0.instance.vtx", "minecraft/region.0.0.instance.cfg" },
+  { "minecraft/region.-1.0.instance.vtx", "minecraft/region.-1.0.instance.cfg" },
+  { "minecraft/region.0.-1.instance.vtx", "minecraft/region.0.-1.instance.cfg" },
+  { "minecraft/region.-1.-1.instance.vtx", "minecraft/region.-1.-1.instance.cfg" },
+};
+
+static unsigned int vertex_array_objects[region_count];
+static unsigned int vertex_buffers[region_count];
+static unsigned int vertex_count[region_count];
+static unsigned int index_buffer;
+static unsigned int per_vertex_buffer;
 
 static const int vertex_size = 8;
+static const int per_vertex_size = (3 + 3 + 2) * 2;
+
+struct instance_cfg {
+  struct {
+    int instance_count;
+    int offset;
+  } cfg[64];
+};
+
+static instance_cfg instance_cfg[region_count];
 
 void load_program()
 {
   unsigned int program = compile_from_files("shader/test.vert",
-                                            "shader/test.geom",
+                                            NULL, //"shader/test.geom",
                                             "shader/test.frag");
 
   location.attrib.position = glGetAttribLocation(program, "Position");
+  location.attrib.normal = glGetAttribLocation(program, "Normal");
+  location.attrib.texture = glGetAttribLocation(program, "Texture");
+
+  location.attrib.block_position = glGetAttribLocation(program, "BlockPosition");
   location.attrib.block_id = glGetAttribLocation(program, "BlockID");
-  location.attrib.configuration = glGetAttribLocation(program, "Configuration");
+  printf("attributes:\n  position %u\n  normal %u\n  texture %u\n  block_position %u\n  block_id %u\n\n",
-  printf("attributes:\n  position %u\n  block_id %u\n  configuration %u\n",
          location.attrib.position,
-         location.attrib.block_id,
+         location.attrib.normal,
-         location.attrib.configuration);
+         location.attrib.texture,
+         location.attrib.block_position,
+         location.attrib.block_id);
 
   location.uniform.transform = glGetUniformLocation(program, "Transform");
   location.uniform.terrain_sampler = glGetUniformLocation(program, "TerrainSampler");
@@ -53,17 +87,10 @@ void load_program()
   test_program = program;
 }
 
-const char * vertex_paths[] = {
-  "minecraft/region.0.0.inst.vtx",
-  "minecraft/region.0.-1.inst.vtx",
-  "minecraft/region.-1.0.inst.vtx",
-  "minecraft/region.-1.-1.inst.vtx",
-};
-
 void load_vertex_buffer(int i)
 {
   int vertex_buffer_data_size;
-  void * vertex_buffer_data = read_file(vertex_paths[i], &vertex_buffer_data_size);
+  void * vertex_buffer_data = read_file(vertex_paths[i].vtx, &vertex_buffer_data_size);
 
   glBindBuffer(GL_ARRAY_BUFFER, vertex_buffers[i]);
   glBufferData(GL_ARRAY_BUFFER, vertex_buffer_data_size, vertex_buffer_data, GL_STATIC_DRAW);
@@ -72,23 +99,65 @@ void load_vertex_buffer(int i)
   free(vertex_buffer_data);
 }
 
-/*
+void load_per_vertex_buffer()
-void load_element_buffer(int i)
+{
+  int vertex_buffer_data_size;
+  void * vertex_buffer_data = read_file("minecraft/per_vertex.vtx", &vertex_buffer_data_size);
+
+  glBindBuffer(GL_ARRAY_BUFFER, per_vertex_buffer);
+  glBufferData(GL_ARRAY_BUFFER, vertex_buffer_data_size, vertex_buffer_data, GL_STATIC_DRAW);
+
+  free(vertex_buffer_data);
+}
+
+void load_index_buffer()
 {
   int index_buffer_data_size;
-  void * index_buffer_data = read_file(index_paths[i], &index_buffer_data_size);
+  void * index_buffer_data = read_file("minecraft/configuration.idx", &index_buffer_data_size);
 
-  glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, index_buffers[i]);
+  glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, index_buffer);
   glBufferData(GL_ELEMENT_ARRAY_BUFFER, index_buffer_data_size, index_buffer_data, GL_STATIC_DRAW);
-  index_count[i] = index_buffer_data_size / 4;
 
   free(index_buffer_data);
 }
-*/
 
-void load_vertex_attributes()
+void load_vertex_attributes(int i)
 {
+  glBindBuffer(GL_ARRAY_BUFFER, per_vertex_buffer);
+
   glVertexAttribPointer(location.attrib.position,
+                        3,
+                        GL_HALF_FLOAT,
+                        GL_FALSE,
+                        per_vertex_size,
+                        (void*)(0)
+                        );
+
+  glVertexAttribPointer(location.attrib.normal,
+                        3,
+                        GL_HALF_FLOAT,
+                        GL_FALSE,
+                        per_vertex_size,
+                        (void*)(6)
+                        );
+
+  glVertexAttribPointer(location.attrib.texture,
+                        2,
+                        GL_HALF_FLOAT,
+                        GL_FALSE,
+                        per_vertex_size,
+                        (void*)(12)
+                        );
+  glEnableVertexAttribArray(location.attrib.position);
+  glEnableVertexAttribArray(location.attrib.normal);
+  glEnableVertexAttribArray(location.attrib.texture);
+  glVertexAttribDivisor(location.attrib.position, 0);
+  glVertexAttribDivisor(location.attrib.normal, 0);
+  glVertexAttribDivisor(location.attrib.texture, 0);
+
+  glBindBuffer(GL_ARRAY_BUFFER, vertex_buffers[i]);
+
+  glVertexAttribPointer(location.attrib.block_position,
                         3,
                         GL_SHORT,
                         GL_FALSE,
@@ -102,34 +171,43 @@ void load_vertex_attributes()
                         vertex_size,
                         (void*)(6)
                         );
-  glVertexAttribPointer(location.attrib.configuration,
+  glEnableVertexAttribArray(location.attrib.block_position);
-                        1,
-                        GL_UNSIGNED_BYTE,
-                        GL_FALSE,
-                        vertex_size,
-                        (void*)(7)
-                        );
-  glEnableVertexAttribArray(location.attrib.position);
   glEnableVertexAttribArray(location.attrib.block_id);
-  glEnableVertexAttribArray(location.attrib.configuration);
+  glVertexAttribDivisor(location.attrib.block_position, 1);
-  glVertexAttribDivisor(location.attrib.position, 1);
   glVertexAttribDivisor(location.attrib.block_id, 1);
-  glVertexAttribDivisor(location.attrib.configuration, 1);
+}
+
+void load_instance_cfg(int i)
+{
+  int data_size;
+  void * data = read_file(vertex_paths[i].cfg, &data_size);
+  assert(data_size == 512);
+
+  memcpy(&instance_cfg[i], data, data_size);
 }
 
 void load_buffers()
 {
-  glGenVertexArrays(4, vertex_array_objects);
+  // per-vertex buffer
-  //glGenBuffers(4, index_buffers);
+  glGenBuffers(1, &per_vertex_buffer);
-  glGenBuffers(4, vertex_buffers);
+  load_per_vertex_buffer();
 
-  for (int i = 0; i < 4; i++) {
+  // per-instance buffer
+  glGenVertexArrays(region_count, vertex_array_objects);
+  glGenBuffers(region_count, vertex_buffers);
+
+  for (int i = 0; i < region_count; i++) {
     glBindVertexArray(vertex_array_objects[i]);
 
-    //load_element_buffer(i);
     load_vertex_buffer(i);
-    load_vertex_attributes();
+    load_vertex_attributes(i);
+    load_instance_cfg(i);
   }
+
+  // index buffer
+
+  glGenBuffers(1, &index_buffer);
+  load_index_buffer();
 }
 
 static unsigned int texture;
@@ -167,8 +245,6 @@ void load_texture_shader_storage()
   void * textures_data = read_file("minecraft/block_id_to_texture_id.data", &textures_data_size);
   assert(textures_data != NULL);
 
-  printf("%d\n", textures_data_size);
-
   glBufferData(GL_UNIFORM_BUFFER, textures_data_size, textures_data, GL_STATIC_DRAW);
   free(textures_data);
 
@@ -205,6 +281,11 @@ void update(float lx, float ly, float ry)
   vz += -2.5 * ly;
 }
 
+static inline int popcount(int x)
+{
+  return __builtin_popcount(x);
+}
+
 void draw()
 {
   XMVECTOR eye = XMVectorSet(vx + -50.0f, vz + -50.0f, vy + 150.0f, 0.0f);
@@ -237,22 +318,24 @@ void draw()
   //glBindBuffer(GL_UNIFORM_BUFFER, textures_ubo);
   //glBindBufferBase(GL_UNIFORM_BUFFER, 0, textures_ubo);
 
-  glEnable(GL_CULL_FACE);
+  //glEnable(GL_CULL_FACE);
-  glCullFace(GL_FRONT);
+  //glCullFace(GL_FRONT);
-  glFrontFace(GL_CCW);
+  //glFrontFace(GL_CCW);
 
-  for (int i = 0; i < 4; i++) {
+  for (int region_index = 0; region_index < region_count; region_index++) {
-    glBindVertexArray(vertex_array_objects[i]);
+    glBindVertexArray(vertex_array_objects[region_index]);
+    glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, index_buffer);
+    for (int configuration = 1; configuration < 64; configuration++) {
+      int element_count = 6 * popcount(configuration);
+      const void * indices = (void *)((ptrdiff_t)index_buffer_configuration_offsets[configuration]); // index into configuration.idx
 
-    //glDrawElements(GL_TRIANGLES, index_count[i], GL_UNSIGNED_INT, 0);
+      int instance_count = instance_cfg[region_index].cfg[configuration].instance_count;
+      int base_instance = instance_cfg[region_index].cfg[configuration].offset / vertex_size; // index into region.0.0.instance.vtx
 
-    int instance_count = vertex_count[i];
+      if (instance_count == 0)
-    //printf("instance_count %d\n", instance_count);
+        continue;
 
-    glPointSize(10.0);
+      glDrawElementsInstancedBaseInstance(GL_TRIANGLES, element_count, GL_UNSIGNED_BYTE, indices, instance_count, base_instance);
-    glDrawArraysInstanced(GL_POINTS,
+    }
-                          0,
-                          1,
-                          instance_count);
   }
 }