audio resampler and visualizer

Makefile

@@ -68,6 +68,8 @@ OBJS = \
 	src/collada/animate.o \
 	src/flame.o \
 	src/audio.o \
+	src/audio/visualizer.o \
+	src/audio/resampler.o \
 	$(COLLADA_SCENES) \
 	$(MINECRAFT_OBJS)
 

include/audio/resampler.h

@@ -0,0 +1,14 @@
+#pragma once
+
+namespace audio::resampler {
+  void load();
+  void compute(unsigned int input_texture, int start, int end);
+
+  struct texture_buffer {
+    unsigned int texture;
+    unsigned int buffer;
+  };
+
+  //extern texture_buffer input;
+  extern texture_buffer output;
+}

include/audio/visualizer.h

@@ -0,0 +1,13 @@
+namespace audio::visualizer {
+
+  void load();
+  void draw(unsigned int vertex_array_object, unsigned int index_buffer);
+
+  struct visualizer_state {
+    int sample_count;
+    int start_offset;
+    int end_offset;
+  };
+
+  extern visualizer_state state;
+}

include/opengl.h

@@ -3,10 +3,21 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+  unsigned int compile(char const * vertex_source,
+                       int vertex_source_size,
+                       char const * geometry_source,
+                       int geometry_source_size,
+                       char const * fragment_source,
+                       int fragment_source_size);
 
-  unsigned int compile_from_files(const char * vertex_path,
+  unsigned int compile_from_files(char const * vertex_path,
-                                  const char * geometry_path,
+                                  char const * geometry_path,
-                                  const char * fragment_path);
+                                  char const * fragment_path);
+
+  unsigned int compile_compute(char const * compute_source,
+                               int compute_source_size);
+
+  unsigned int compile_compute_from_files(char const * compute_path);
 
   unsigned int load_uniform_buffer(char const * const path, int * out_size);
 

shader/audio/resampler.comp

@@ -0,0 +1,27 @@
+#version 430 core
+
+layout (local_size_x = 128, local_size_y = 1, local_size_z = 1) in;
+
+layout (r16i, binding = 0) readonly uniform iimageBuffer ImageIn;
+layout (r16i, binding = 1) writeonly uniform iimageBuffer ImageOut;
+
+layout (location = 0) uniform int GlobalOffset;
+layout (location = 1) uniform int WindowLength;
+
+void main()
+{
+  int x = int(gl_GlobalInvocationID.x);
+
+  int offset = GlobalOffset + x * WindowLength;
+
+  float sum = 0;
+  //int sum = 0;
+  float scale = 1.0 / float(WindowLength);
+  for (int i = 0; i < WindowLength; i++) {
+    ivec4 v = imageLoad(ImageIn, offset + i);
+    sum += float(abs(v.x)) * scale;
+    //sum = max(abs(v.x), sum);
+  }
+
+  imageStore(ImageOut, x, ivec4(int(sum)));
+}

shader/audio/visualizer.frag

@@ -0,0 +1,26 @@
+#version 430 core
+
+in VS_OUT {
+  vec4 Texture;
+} fs_in;
+
+layout (location = 0) out vec4 Color;
+
+layout (r16i, binding = 0) readonly uniform iimageBuffer Image;
+
+float get_sample(int coordinate)
+{
+  int red = imageLoad(Image, coordinate).r;
+  return float(red) * (1.0 / 32768.0);
+}
+
+void main()
+{
+  float value = get_sample(int(gl_FragCoord.x));
+
+  float y = -(fs_in.Texture.y * 2 - 1);
+
+  float c = float(abs(y) < abs(value));
+
+  Color = vec4(c, 0, 0, 1);
+}

shader/audio/visualizer.vert

@@ -0,0 +1,19 @@
+#version 430 core
+
+const vec2 vtx[4] = vec2[](vec2(-1.0,  1.0),  // tl
+                           vec2( 1.0,  1.0),  // tr
+                           vec2( 1.0, -1.0),  // br
+                           vec2(-1.0, -1.0)); // bl
+
+out VS_OUT {
+  vec4 Texture;
+} vs_out;
+
+void main()
+{
+  vec2 vertex = vtx[gl_VertexID];
+
+  vs_out.Texture = vec4(vertex * vec2(0.5, -0.5) + 0.5, 0, 0);
+
+  gl_Position = vec4(vertex, 0, 1.0);
+}

src/audio/resampler.cpp

@@ -0,0 +1,154 @@
+#include <stdio.h>
+#include <assert.h>
+
+#include "glad/gl.h"
+
+#include "opengl.h"
+#include "audio/resampler.h"
+#include "window.h"
+
+namespace audio::resampler {
+
+  unsigned int program;
+
+  //texture_buffer input;
+  texture_buffer output;
+
+  static const int max_output_pixels = 4096;
+  static const int local_size = 128;
+
+  static void load_max_compute()
+  {
+    int max_compute_work_group_count[3];
+    int max_compute_work_group_size[3];
+    int max_compute_work_group_invocations;
+
+    for (int i = 0; i < 3; i++) {
+      glGetIntegeri_v(GL_MAX_COMPUTE_WORK_GROUP_COUNT, i, &max_compute_work_group_count[i]);
+      glGetIntegeri_v(GL_MAX_COMPUTE_WORK_GROUP_SIZE, i, &max_compute_work_group_size[i]);
+    }
+    glGetIntegerv(GL_MAX_COMPUTE_WORK_GROUP_INVOCATIONS, &max_compute_work_group_invocations);
+
+    printf("GL_MAX_COMPUTE_WORK_GROUP_COUNT %d %d %d\n",
+           max_compute_work_group_count[0],
+           max_compute_work_group_count[1],
+           max_compute_work_group_count[2]);
+
+    printf("GL_MAX_COMPUTE_WORK_GROUP_SIZE %d %d %d\n",
+           max_compute_work_group_size[0],
+           max_compute_work_group_size[1],
+           max_compute_work_group_size[2]);
+
+    printf("GL_MAX_COMPUTE_WORK_GROUP_INVOCATIONS %d\n", max_compute_work_group_invocations);
+  }
+
+  static void load_shader()
+  {
+    program = compile_compute_from_files("shader/audio/resampler.comp");
+  }
+
+  static void load_texture_buffers()
+  {
+    /*
+    int16_t data[128];
+    for (int i = 0; i < 128; i++) {
+      data[i] = i + 1;
+    }
+
+    // input
+    glGenBuffers(1, &input.buffer);
+    glGenTextures(1, &input.texture);
+    glBindBuffer(GL_TEXTURE_BUFFER, input.buffer);
+    glBufferData(GL_TEXTURE_BUFFER, (sizeof (data)), data, GL_STATIC_DRAW);
+    glBindBuffer(GL_TEXTURE_BUFFER, 0);
+
+    glBindTexture(GL_TEXTURE_BUFFER, input.texture);
+    glTexBuffer(GL_TEXTURE_BUFFER, GL_R16I, input.buffer);
+    glBindTexture(GL_TEXTURE_BUFFER, 0);
+    */
+
+    // output
+    int output_size = max_output_pixels * 2;
+    glGenBuffers(1, &output.buffer);
+    glGenTextures(1, &output.texture);
+    glBindBuffer(GL_TEXTURE_BUFFER, output.buffer);
+    glBufferData(GL_TEXTURE_BUFFER, output_size, NULL, GL_DYNAMIC_DRAW);
+    glBindBuffer(GL_TEXTURE_BUFFER, 0);
+
+    glBindTexture(GL_TEXTURE_BUFFER, output.texture);
+    glTexBuffer(GL_TEXTURE_BUFFER, GL_R16I, output.buffer);
+    glBindTexture(GL_TEXTURE_BUFFER, 0);
+  }
+
+  void load()
+  {
+    load_max_compute();
+
+    load_shader();
+
+    load_texture_buffers();
+  }
+
+  void compute(unsigned int input_texture, int start, int end)
+  {
+    glUseProgram(program);
+
+    //
+    assert(window::width < max_output_pixels);
+    int count = end - start;
+    if (count < window::width) {
+      int center = (end + start) / 2;
+      start = center - (window::width / 2);
+      end = center + (window::width / 2);
+      assert(end > start);
+      count = end - start;
+      if (count < window::width) {
+        end += 1;
+        count = end - start;
+      }
+      assert(count == window::width);
+    }
+
+    int window_length = count / window::width;
+
+    glUniform1i(0, start);
+    glUniform1i(1, window_length);
+
+    //
+
+    glBindImageTexture(0,
+                       input_texture,
+                       0,
+                       GL_FALSE,
+                       0,
+                       GL_READ_ONLY,
+                       GL_R16I);
+
+    glBindImageTexture(1,
+                       output.texture,
+                       0,
+                       GL_FALSE,
+                       0,
+                       GL_WRITE_ONLY,
+                       GL_R16I);
+
+    int groups = window::width / local_size;
+    if (groups == 0)
+      groups = 1;
+    glDispatchCompute(groups, 1, 1);
+
+    glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT);
+
+    /*
+    glBindBuffer(GL_TEXTURE_BUFFER, output.buffer);
+    void const * buf = glMapBufferRange(GL_TEXTURE_BUFFER, 0, (2 * 128), GL_MAP_READ_BIT);
+    int16_t const * data = (int16_t const *)buf;
+
+    for (int i = 0; i < 128; i++) {
+      printf("%d: %d\n", i, data[i]);
+    }
+
+    glUnmapBuffer(GL_TEXTURE_BUFFER);
+    */
+  }
+}

src/audio/visualizer.cpp

@@ -0,0 +1,86 @@
+#include <assert.h>
+#include <stdio.h>
+#include <math.h>
+
+#include "glad/gl.h"
+
+#include "opengl.h"
+#include "file.h"
+#include "window.h"
+
+#include "audio/visualizer.h"
+#include "audio/resampler.h"
+
+namespace audio::visualizer {
+
+  static unsigned int program;
+  static unsigned int texture;
+  static unsigned int buffer;
+
+  visualizer_state state;
+
+  static void load_shader()
+  {
+    program = compile_from_files("shader/audio/visualizer.vert",
+                                 nullptr, // geom
+                                 "shader/audio/visualizer.frag");
+  }
+
+  static void load_audio()
+  {
+    int size;
+    void const * data = file::read_file("audio/Suite.pcm", &size);
+    //void const * data = file::read_file("/home/bilbo/Documents/Sine.raw", &size);
+    assert(data != nullptr);
+
+    int max_texture_buffer_size;
+    glGetIntegerv(GL_MAX_TEXTURE_BUFFER_SIZE, &max_texture_buffer_size);
+    assert(max_texture_buffer_size >= size);
+
+    glGenBuffers(1, &buffer);
+    glBindBuffer(GL_TEXTURE_BUFFER, buffer);
+    glBufferData(GL_TEXTURE_BUFFER, size, data, GL_STATIC_DRAW);
+    glBindBuffer(GL_TEXTURE_BUFFER, 0);
+
+    glGenTextures(1, &texture);
+    glBindTexture(GL_TEXTURE_BUFFER, texture);
+    glTexBuffer(GL_TEXTURE_BUFFER, GL_R16I, buffer);
+    glBindTexture(GL_TEXTURE_BUFFER, 0);
+
+    state.sample_count = size / 2;
+    state.start_offset = 0;
+    state.end_offset = state.sample_count;
+  }
+
+  void load()
+  {
+    load_shader();
+    load_audio();
+  }
+
+  void draw(unsigned int vertex_array_object, unsigned int index_buffer)
+  {
+    if (state.start_offset < 0)
+      state.start_offset = 0;
+    if (state.end_offset > state.sample_count)
+      state.end_offset = state.sample_count;
+
+    resampler::compute(texture, state.start_offset, state.end_offset);
+
+    glUseProgram(program);
+    glDepthFunc(GL_ALWAYS);
+
+    glBindVertexArray(vertex_array_object);
+    glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, index_buffer);
+
+    glBindImageTexture(0,
+                       resampler::output.texture,
+                       0,
+                       GL_FALSE,
+                       0,
+                       GL_READ_ONLY,
+                       GL_R16I);
+
+    glDrawElements(GL_TRIANGLE_STRIP, 4, GL_UNSIGNED_BYTE, (void *)0);
+  }
+}

src/hud.cpp

@@ -9,6 +9,7 @@
 #include "view.h"
 #include "window.h"
 #include "test.h"
+#include "audio/visualizer.h"
 
 // depends on:
 // - font::bitmap::load
@@ -110,13 +111,15 @@ namespace hud {
     y = draw_label<float>(ter_best, buf, 10, y, "pitch: ", "%.4f", view::state.pitch);
     //y = draw_label<float>(ter_best, buf, 10, y, "frame_rate_avg: ", "%.2f", 1.0f / update_average(current_time - last_frame_time));
 
+    /*
     font::bitmap::draw_string(ter_best, "mouse:", 10, y); y += ter_best.desc->glyph_height;
 
     y = draw_vector(ter_best, buf, y, "  position", XMLoadFloat4((XMFLOAT4*)test::mouse_position));
     y = draw_vector(ter_best, buf, y, "     block", XMLoadFloat4((XMFLOAT4*)test::mouse_block));
+    */
 
-    if (frame++ > 60 * 10)
+    y = draw_label<int>(ter_best, buf, 10, y, "start: ", "%d", audio::visualizer::state.start_offset);
-      return;
+    y = draw_label<int>(ter_best, buf, 10, y, "end: ", "%d", audio::visualizer::state.end_offset);
 
     /*
     font::outline::draw_start(uncial_antiqua_fonts[0], empty_vertex_array_object, quad_index_buffer);

src/main.cpp

@@ -147,7 +147,9 @@ int main()
           goto exit;
         break;
       case SDL_EVENT_WINDOW_RESIZED:
-        printf("%d %d\n", event.window.data1, event.window.data2);
+        printf("resize %d %d\n", event.window.data1, event.window.data2);
+        update_window(event.window.data1, event.window.data2);
+        glViewport(0, 0, event.window.data1, event.window.data2);
         break;
       case SDL_EVENT_GAMEPAD_ADDED:
         add_gamepad(event.gdevice.which);

src/opengl.cpp

@@ -9,6 +9,9 @@
 #include "file.h"
 #include "dds_validate.h"
 
+static const int max_log_size = 4096;
+static char info_log[max_log_size];
+
 unsigned int compile(char const * vertex_source,
                      int vertex_source_size,
                      char const * geometry_source,
@@ -17,7 +20,6 @@ unsigned int compile(char const * vertex_source,
                      int fragment_source_size)
 {
   int compile_status;
-  char info_log[512];
 
   // program
   unsigned int shader_program = glCreateProgram();
@@ -31,8 +33,9 @@ unsigned int compile(char const * vertex_source,
   glCompileShader(vertex_shader);
   glGetShaderiv(vertex_shader, GL_COMPILE_STATUS, &compile_status);
   if (!compile_status) {
-    glGetShaderInfoLog(vertex_shader, 512, NULL, info_log);
+    glGetShaderInfoLog(vertex_shader, max_log_size, nullptr, info_log);
     fprintf(stderr, "vertex shader compile: %s\n", info_log);
+    assert(!"vertex shader compile");
   }
   glAttachShader(shader_program, vertex_shader);
 
@@ -43,8 +46,9 @@ unsigned int compile(char const * vertex_source,
     glCompileShader(geometry_shader);
     glGetShaderiv(geometry_shader, GL_COMPILE_STATUS, &compile_status);
     if (!compile_status) {
-      glGetShaderInfoLog(geometry_shader, 512, NULL, info_log);
+      glGetShaderInfoLog(geometry_shader, max_log_size, nullptr, info_log);
       fprintf(stderr, "geometry shader compile: %s\n", info_log);
+      assert(!"geometry shader compile");
     }
     glAttachShader(shader_program, geometry_shader);
   }
@@ -55,8 +59,9 @@ unsigned int compile(char const * vertex_source,
   glCompileShader(fragment_shader);
   glGetShaderiv(fragment_shader, GL_COMPILE_STATUS, &compile_status);
   if (!compile_status) {
-    glGetShaderInfoLog(fragment_shader, 512, NULL, info_log);
+    glGetShaderInfoLog(fragment_shader, max_log_size, nullptr, info_log);
     fprintf(stderr, "fragment shader compile: %s\n", info_log);
+    assert(!"fragment shader compile");
   }
   glAttachShader(shader_program, fragment_shader);
 
@@ -64,8 +69,9 @@ unsigned int compile(char const * vertex_source,
   glLinkProgram(shader_program);
   glGetProgramiv(shader_program, GL_LINK_STATUS, &compile_status);
   if (!compile_status) {
-    glGetProgramInfoLog(shader_program, 512, NULL, info_log);
+    glGetProgramInfoLog(shader_program, max_log_size, nullptr, info_log);
     fprintf(stderr, "shader link: %s\n", info_log);
+    assert(!"shader link");
   }
 
   glDeleteShader(vertex_shader);
@@ -81,22 +87,22 @@ unsigned int compile_from_files(char const * vertex_path,
                                 char const * fragment_path)
 {
   int vertex_source_size = 0;
-  void const * vertex_source = NULL;
+  void const * vertex_source = nullptr;
   int geometry_source_size = 0;
-  void const * geometry_source = NULL;
+  void const * geometry_source = nullptr;
   int fragment_source_size = 0;
-  void const * fragment_source = NULL;
+  void const * fragment_source = nullptr;
 
   vertex_source = file::read_file(vertex_path, &vertex_source_size);
-  assert(vertex_source != NULL);
+  assert(vertex_source != nullptr);
 
-  if (geometry_path != NULL) {
+  if (geometry_path != nullptr) {
     geometry_source = file::read_file(geometry_path, &geometry_source_size);
-    assert(geometry_source != NULL);
+    assert(geometry_source != nullptr);
   }
 
   fragment_source = file::read_file(fragment_path, &fragment_source_size);
-  assert(fragment_source != NULL);
+  assert(fragment_source != nullptr);
 
   unsigned int program = compile((char const *)vertex_source, vertex_source_size,
                                  (char const *)geometry_source, geometry_source_size,
@@ -109,6 +115,53 @@ unsigned int compile_from_files(char const * vertex_path,
   return program;
 }
 
+unsigned int compile_compute(char const * compute_source,
+                             int compute_source_size)
+{
+  int compile_status;
+
+  unsigned int shader_program = glCreateProgram();
+
+  // compute shader
+  unsigned int compute_shader = glCreateShader(GL_COMPUTE_SHADER);
+  glShaderSource(compute_shader, 1, &compute_source, &compute_source_size);
+  glCompileShader(compute_shader);
+  glGetShaderiv(compute_shader, GL_COMPILE_STATUS, &compile_status);
+  if (!compile_status) {
+    glGetShaderInfoLog(compute_shader, max_log_size, nullptr, info_log);
+    fprintf(stderr, "compute shader compile: %s\n", info_log);
+    assert(!"compute shader compile");
+  }
+  glAttachShader(shader_program, compute_shader);
+
+  // link shaders
+  glLinkProgram(shader_program);
+  glGetProgramiv(shader_program, GL_LINK_STATUS, &compile_status);
+  if (!compile_status) {
+    glGetProgramInfoLog(shader_program, max_log_size, nullptr, info_log);
+    fprintf(stderr, "compute shader link: %s\n", info_log);
+    assert(!"compute shader link");
+  }
+
+  glDeleteShader(compute_shader);
+
+  return shader_program;
+}
+
+unsigned int compile_compute_from_files(char const * compute_path)
+{
+  int compute_source_size = 0;
+  void const * compute_source = nullptr;
+
+  compute_source = file::read_file(compute_path, &compute_source_size);
+
+  unsigned int program = compile_compute((char const *)compute_source, compute_source_size);
+
+  file::free(compute_source);
+
+  return program;
+}
+
 unsigned int load_uniform_buffer(char const * const path, int * out_size)
 {
   unsigned int buffer;
@@ -117,7 +170,7 @@ unsigned int load_uniform_buffer(char const * const path, int * out_size)
 
   int data_size;
   void const * data = file::read_file(path, &data_size);
-  assert(data != NULL);
+  assert(data != nullptr);
 
   glBufferData(GL_UNIFORM_BUFFER, data_size, data, GL_STATIC_DRAW);
   file::free(data);
@@ -135,7 +188,7 @@ void load_dds_texture_2D(char const * const path)
 
   int size;
   void const * data = file::read_file(path, &size);
-  assert(data != NULL);
+  assert(data != nullptr);
 
   void * image_data;
   int image_size;

src/test.cpp

@@ -31,6 +31,8 @@
 #include "flame.h"
 #include "new.h"
 #include "popcount.h"
+#include "audio/visualizer.h"
+#include "audio/resampler.h"
 
 #include "world/entry_table.h"
 #include "world/world.h"
@@ -209,6 +211,14 @@ void load()
   flame::load_program();
   flame::load_texture();
 
+  //////////////////////////////////////////////////////////////////////
+  // audio visualizer
+  //////////////////////////////////////////////////////////////////////
+
+  audio::visualizer::load();
+
+  audio::resampler::load();
+
   //////////////////////////////////////////////////////////////////////
   // opengl state
   //////////////////////////////////////////////////////////////////////
@@ -400,6 +410,9 @@ void update_joystick(float lx, float ly,
   if (lighting.linear < 0.0f)
     lighting.linear = 0.0f;
   */
+
+  audio::visualizer::state.start_offset += lx * 20000.0;
+  audio::visualizer::state.end_offset += rx * 20000.0;
 }
 
 static float ext_position = 0;
@@ -484,7 +497,7 @@ void draw()
   glClearColor(0.0f, 0.0f, 0.0f, 0.0f);
   glClearDepth(-1.0f);
 
-  if (true) {
+  if (false) {
     // possibly re-initialize geometry buffer if window width/height changes
     init_geometry_buffer(geometry_buffer_pnc, geometry_buffer_pnc_types);
 
@@ -525,6 +538,7 @@ void draw()
     glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
     //scene_state.draw();
 
+    audio::visualizer::draw(empty_vertex_array_object, quad_index_buffer);
     hud::draw();
   }