audio: looped tracks

.gitignore

@@ -10,3 +10,5 @@ tool/pack_file
 *.dds
 *.zlib
 tools/compress
+tools/opus_encode
+*.pcm

Makefile

@@ -23,6 +23,10 @@ CFLAGS += -fno-strict-aliasing
 CFLAGS += -I./include
 CFLAGS += -I./data
 CFLAGS += -I../SDL3-dist/include
+ifeq ($(UNAME),Darwin)
+CFLAGS += -I../MoltenVK/MoltenVK/include
+endif
+CFLAGS += -I../opus-dist/include
 CFLAGS += -fpic
 CFLAGS += -ffunction-sections
 CFLAGS += -fdata-sections
@@ -63,7 +67,8 @@ OBJS = \
 	src/renpy/vulkan.o \
 	src/renpy/script.o \
 	src/renpy/interpreter.o \
-	src/renpy/interact.o
+	src/renpy/interact.o \
+	src/audio.o
 
 ifeq ($(UNAME),Linux)
 ZLIB = ../zlib-1.3.2
@@ -84,7 +89,8 @@ LIBS = \
 	 ../SDL3-dist/lib/libSDL3.a
 else
 LIBS = \
-	 ../SDL3-dist/lib64/libSDL3.a
+	 ../SDL3-dist/lib64/libSDL3.a \
+	../opus-dist/lib/libopus.a
 endif
 
 all: main
@@ -101,6 +107,12 @@ all: main
 #%.dds: %.png
 #	WINEDEBUG=-all wine $(HOME)/Texconv.exe -y -nogpu -nowic -dx10 --format BC7_UNORM_SRGB -m 1 $< -o $(dir $@)
 
+%.pcm: %.wav
+	ffmpeg -loglevel quiet -y -i $< -c:a pcm_s16le -ar 48000 -ac 2 -f s16le $@
+
+%.opus.bin: %.pcm
+	./tools/opus_encode $< $@
+
 main: $(OBJS) $(LIBS)
 	$(CC) $(ARCH) $(LDFLAGS) $(FLAGS) $(OPT) $(DEBUG) $^ -o $@
 

filenames.txt

@@ -11,3 +11,5 @@ data/renpy/images/ch/cat.dds
 data/renpy/images/ch/catw.dds
 data/renpy/images/ch/Eily.dds
 data/renpy/images/ch/Alice.dds
+
+audio/PhrygianButterflies.opus.bin

include/audio.h

@@ -0,0 +1,7 @@
+#pragma once
+
+namespace audio {
+  void init();
+  void load();
+  void update();
+}

src/audio.cpp

@@ -0,0 +1,192 @@
+#include <assert.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include <opus/opus.h>
+#include <SDL3/SDL.h>
+
+#include "file.h"
+#include "audio.h"
+#include "new.h"
+#include "minmax.h"
+
+namespace audio {
+
+  int const frame_samples = 960; // 20 milliseconds @ 48kHz
+  int const sample_rate = 48000;
+  int const channels = 2;
+  int const sample_size = (sizeof (int16_t));
+
+  int const max_frame_size = 960 * 3; // 20ms at 48kHz
+  int const max_packet_size = 1275;
+
+  //
+
+  SDL_AudioStream * audio_stream;
+  SDL_AudioSpec audio_spec;
+
+  OpusDecoder * opus_decoder;
+
+  struct AudioFile {
+    char const * const path;
+    uint32_t loop_end;
+  };
+
+  struct AudioBuffer {
+    AudioFile * audio_file;
+    int16_t * buf;
+    uint32_t sample_count;
+  };
+
+  struct AudioInstance {
+    AudioBuffer * audio_buffer;
+    uint32_t sample_index;
+    uint32_t tail_index;
+  };
+
+  consteval uint32_t time_to_samples(double m, double s)
+  {
+    return (m * 60.0 + s) * sample_rate;
+  }
+
+  AudioFile audio_files[] = {
+    {
+      .path = "audio/PhrygianButterflies.opus.bin",
+      .loop_end = time_to_samples(0, 40.125),
+    },
+  };
+  constexpr int audio_files_count = (sizeof (audio_files)) / (sizeof (audio_files[0]));
+  AudioBuffer audio_buffers[audio_files_count];
+
+  constexpr int max_audio_instances = 16;
+  AudioInstance audio_instances[max_audio_instances];
+
+  void init()
+  {
+    audio_spec.channels = channels;
+    audio_spec.format = SDL_AUDIO_S16LE;
+    audio_spec.freq = sample_rate;
+    audio_stream = SDL_OpenAudioDeviceStream(SDL_AUDIO_DEVICE_DEFAULT_PLAYBACK, &audio_spec, NULL, NULL);
+    assert(audio_stream);
+    SDL_ResumeAudioStreamDevice(audio_stream);
+
+    int err;
+    opus_decoder = opus_decoder_create(sample_rate, channels, &err);
+    if (err < 0) {
+      fprintf(stderr, "opus_decoder_create: %s\n", opus_strerror(err));
+      assert(!"opus_decoder_create");
+    }
+  }
+
+  void decode(char const * const filename, AudioBuffer * audio_buffer)
+  {
+    uint32_t size;
+    uint8_t const * buf = (uint8_t const *)file::open(filename, &size);
+    assert(buf != nullptr);
+
+    uint32_t samples_count = (buf[3] << 24)
+                           | (buf[2] << 16)
+                           | (buf[1] << 8)
+                           | (buf[0] << 0);
+    uint32_t offset = 4;
+    uint32_t samples_decoded = 0;
+    uint32_t samples_copied = 0;
+
+    int err = opus_decoder_ctl(opus_decoder, OPUS_RESET_STATE);
+    if (err < 0) {
+      fprintf(stderr, "opus_encoder_ctl(OPUS_RESET_STATE): %s\n", opus_strerror(err));
+      assert(!"opus_encoder_ctl");
+    }
+
+    int16_t * output_buf = NewM<int16_t>(channels * samples_count);
+
+    // decode packets
+    while (offset < size) {
+      uint16_t packet_size = (buf[offset + 1] << 8) | (buf[offset + 0] << 0);
+      offset += 2;
+
+      assert(offset + packet_size <= size);
+      int16_t decode_buf[max_frame_size * channels];
+      int frame_size = opus_decode(opus_decoder, &buf[offset], packet_size, decode_buf, max_frame_size, 0);
+      if (frame_size < 0) {
+        fprintf(stderr, "opus_decode: %s\n", opus_strerror(frame_size));
+        assert(!"opus_decode\n");
+      }
+      samples_decoded += frame_size;
+
+      uint32_t copy_samples = min(samples_decoded, samples_count) - samples_copied;
+      memcpy(&output_buf[samples_copied * channels], decode_buf, copy_samples * channels * sample_size);
+
+      samples_copied += copy_samples;
+      offset += packet_size;
+    }
+
+    printf("copied %d decoded %d count %d\n", samples_copied, samples_decoded, samples_count);
+    assert(samples_decoded >= samples_count);
+    assert(samples_copied == samples_count);
+
+    audio_buffer->buf = output_buf;
+    audio_buffer->sample_count = samples_count;
+
+    assert(audio_buffer->sample_count / 2);
+  }
+
+  void load()
+  {
+    for (int i = 0; i < audio_files_count; i++) {
+      audio_buffers[i].audio_file = &audio_files[i];
+      decode(audio_files[i].path, &audio_buffers[i]);
+      audio_instances[i].audio_buffer = &audio_buffers[i];
+      audio_instances[i].sample_index = 0;
+      audio_instances[i].tail_index = audio_buffers[i].sample_count;
+    }
+  }
+
+  inline static int min(int a, int b)
+  {
+    return (a < b) ? a : b;
+  }
+
+  void update()
+  {
+    int half_period_samples = audio_spec.freq / 2;
+    int half_period_size = half_period_samples * sample_size * audio_spec.channels;
+    if (SDL_GetAudioStreamQueued(audio_stream) >= half_period_size)
+      return;
+
+    int16_t mix_buffer[half_period_samples * channels];
+    memset(mix_buffer, 0, (sizeof (mix_buffer)));
+
+    AudioInstance & instance = audio_instances[0];
+    int16_t const * const buf = instance.audio_buffer->buf;
+    uint32_t const sample_count = instance.audio_buffer->sample_count;
+    uint32_t const loop_end = instance.audio_buffer->audio_file->loop_end;
+
+    uint32_t mix_index = 0;
+    for (int i = 0; i < half_period_samples; i++) {
+      if (instance.sample_index >= loop_end) {
+        instance.sample_index = 0;
+        instance.tail_index = loop_end;
+        fprintf(stderr, "loop\n");
+      }
+
+      assert(instance.sample_index < sample_count);
+      assert(instance.tail_index <= sample_count);
+
+      for (int ch = 0; ch < channels; ch++) {
+        mix_buffer[mix_index * channels + ch] += buf[instance.sample_index * channels + ch];
+        if (instance.tail_index != sample_count) {
+          mix_buffer[mix_index * channels + ch] += buf[instance.tail_index * channels + ch];
+        }
+      }
+      instance.sample_index += 1;
+      if (instance.tail_index != sample_count) {
+        instance.tail_index += 1;
+      }
+
+      mix_index += 1;
+    }
+
+    SDL_PutAudioStreamData(audio_stream, (void *)mix_buffer, half_period_size);
+  }
+}

src/main.cpp

@@ -28,6 +28,8 @@
 #include "scenes/shadow_test/shadow_test.h"
 #include "scenes/eidelwind/eidelwind.h"
 
+#include "audio.h"
+
 VkInstance instance{ VK_NULL_HANDLE };
 VkDevice device{ VK_NULL_HANDLE };
 VkQueue queue{ VK_NULL_HANDLE };
@@ -357,7 +359,8 @@ int main()
 {
   file::init();
 
-  SDL_CHECK(SDL_Init(SDL_INIT_VIDEO | SDL_INIT_GAMEPAD));
+  SDL_InitFlags init_flags = SDL_INIT_VIDEO | SDL_INIT_GAMEPAD | SDL_INIT_AUDIO;
+  SDL_CHECK(SDL_Init(init_flags));
   SDL_CHECK(SDL_Vulkan_LoadLibrary(NULL));
   volkInitialize();
 
@@ -768,7 +771,11 @@ int main()
 
   //collada_state.update(0);
 
+  audio::init();
+  audio::load();
+
   while (quit == false) {
+    audio::update();
     interpreter_state.interpret();
 
     SDL_Event event;
@@ -1216,7 +1223,7 @@ int main()
   vkDestroyCommandPool(device, commandPool, nullptr);
 
   SDL_DestroyWindow(window);
-  SDL_QuitSubSystem(SDL_INIT_VIDEO);
+  SDL_QuitSubSystem(init_flags);
   SDL_Quit();
 
   vkDestroyDevice(device, nullptr);

tools/Makefile

@@ -0,0 +1,9 @@
+OPT = -O2
+CFLAGS = -I../../opus-dist/include
+CFLAGS = -I../include
+
+opus_encode: opus_encode.c ../../opus-dist/lib/libopus.a
+	gcc -o $@ $(OPT) $(CFLAGS) -lm $^
+
+pack_file: pack_file.cpp
+	g++ -o $@ $(OPT) $(CFLAGS) $^

tools/opus_encode.c

@@ -0,0 +1,121 @@
+#include <assert.h>
+#include <errno.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <opus/opus.h>
+
+const int frame_size = 960; // 20ms at 48kHz
+const int sample_rate = 48000;
+const int channels = 2;
+const int bitrate = 128000;
+
+const int max_packet_size = 3 * 1275;
+
+int main(int argc, char *argv[])
+{
+  assert(argc == 3);
+  int err;
+
+  OpusEncoder * encoder = opus_encoder_create(sample_rate, channels, OPUS_APPLICATION_AUDIO, &err);
+  if (err < 0) {
+    fprintf(stderr, "opus_encoder_create: %s\n", opus_strerror(err));
+    return 1;
+  }
+
+  err = opus_encoder_ctl(encoder, OPUS_SET_BITRATE(bitrate));
+  if (err < 0) {
+    fprintf(stderr, "opus_encoder_ctl(OPUS_SET_BITRATE): %s\n", opus_strerror(err));
+    return 1;
+  }
+
+  char const * input_filename = argv[1];
+  FILE * input_file = fopen(input_filename, "rb");
+  if (input_file == NULL) {
+    fprintf(stderr, "fopen(%s): %s\n", input_filename, strerror(errno));
+    return 1;
+  }
+
+  char const * output_filename = argv[2];
+  FILE * output_file = fopen(output_filename, "wb");
+  if (input_file == NULL) {
+    fprintf(stderr, "fopen(%s): %s\n", output_filename, strerror(errno));
+    return 1;
+  }
+
+  int16_t input[frame_size * channels];
+  uint8_t input_buf[frame_size * channels * (sizeof (int16_t))];
+  uint8_t encode_buf[max_packet_size];
+
+  int total_samples = 0;
+  size_t header_write_1 = fwrite(&total_samples, 1, (sizeof (int)), output_file);
+  if (header_write_1 != (sizeof (int))) {
+    fprintf(stderr, "fwrite: %s\n", strerror(errno));
+    return 1;
+  }
+
+  bool more_data = true;
+  while (more_data) {
+    size_t samples = fread(input_buf, (sizeof (int16_t)) * channels, frame_size, input_file);
+    if (samples == 0)
+      break;
+
+    total_samples += samples;
+
+    if (samples != frame_size) {
+      printf("padding short frame %ld\n", samples);
+      more_data = false;
+      for (int i = samples * channels; i < frame_size * channels; i++) {
+        input_buf[i * 2 + 0] = 0;
+        input_buf[i * 2 + 1] = 0;
+      }
+    }
+    samples = frame_size;
+
+    for (int i = 0; i < channels * frame_size; i++) {
+      // load little endian
+      input[i] = (input_buf[2 * i + 1] << 8) | (input_buf[2 * i + 0] << 0);
+    }
+
+    int bytes = opus_encode(encoder, input, frame_size, encode_buf, max_packet_size);
+    if (bytes < 0) {
+      fprintf(stderr, "opus_encode: %s\n", opus_strerror(bytes));
+      return 1;
+    }
+    assert(bytes <= 1275);
+
+    int16_t encode_bytes = bytes;
+    size_t bytes_write = fwrite(&encode_bytes, 1, (sizeof (encode_bytes)), output_file);
+    if (bytes_write != (sizeof (encode_bytes))) {
+      fprintf(stderr, "fwrite: %s\n", strerror(errno));
+      return 1;
+    }
+    size_t buf_write = fwrite(encode_buf, 1, bytes, output_file);
+    if (buf_write != bytes) {
+      fprintf(stderr, "fwrite: %s\n", strerror(errno));
+      return 1;
+    }
+  }
+
+  fflush(output_file);
+
+  int ret = fseek(output_file, SEEK_SET, 0);
+  if (ret != 0) {
+    fprintf(stderr, "fseek: %s\n", strerror(errno));
+    return 1;
+  }
+
+  size_t header_write_2 = fwrite(&total_samples, 1, (sizeof (int)), output_file);
+  if (header_write_2 != (sizeof (int))) {
+    fprintf(stderr, "fwrite: %s\n", strerror(errno));
+    return 1;
+  }
+  printf("total_samples: %d\n", total_samples);
+
+  fclose(output_file);
+  fclose(input_file);
+
+  return 0;
+}