self-compression

.gitignore

@@ -7,4 +7,6 @@ tool/pack_file
 *.zip
 *.tar
 *.pyc
-*.dds
+*.dds
+*.zlib
+tools/compress

Makefile

@@ -65,13 +65,19 @@ OBJS = \
 	src/renpy/interpreter.o \
 	src/renpy/interact.o
 
-WORLDS = \
+ifeq ($(UNAME),Linux)
-	data/minecraft/midnightmeadow/inthash.o \
+ZLIB = ../zlib-1.3.2
-	data/minecraft/grandlecturn/inthash.o
+CFLAGS += -I$(ZLIB)
-
+OBJS += \
-SCENES = \
+	$(ZLIB)/uncompr.o \
-	data/scenes/shadow_test/shadow_test.o \
+	$(ZLIB)/inflate.o \
-	data/scenes/eidelwind/eidelwind.o
+	$(ZLIB)/inffast.o \
+	$(ZLIB)/inftrees.o \
+	$(ZLIB)/trees.o \
+	$(ZLIB)/zutil.o \
+	$(ZLIB)/crc32.o \
+	$(ZLIB)/adler32.o
+endif
 
 ifeq ($(UNAME),Darwin)
 LIBS = \
@@ -95,7 +101,7 @@ all: main
 #%.dds: %.png
 #	WINEDEBUG=-all wine $(HOME)/Texconv.exe -y -nogpu -nowic -dx10 --format BC7_UNORM_SRGB -m 1 $< -o $(dir $@)
 
-main: $(OBJS) $(LIBS) $(SCENES) $(WORLDS)
+main: $(OBJS) $(LIBS)
 	$(CC) $(ARCH) $(LDFLAGS) $(FLAGS) $(OPT) $(DEBUG) $^ -o $@
 
 %.spv: %.hlsl
@@ -104,7 +110,10 @@ main: $(OBJS) $(LIBS) $(SCENES) $(WORLDS)
 tool/pack_file: tool/pack_file.cpp
 	make -C tool pack_file
 
-src/pack.o: files.pack
+%.pack.zlib: %.pack
+	./tools/compress $< $@
+
+src/pack.o: files.pack.zlib
 
 PACK_FILENAMES = $(shell cat filenames.txt)
 files.pack: tool/pack_file $(PACK_FILENAMES) filenames.txt

include/file.h

@@ -3,6 +3,8 @@
 #include <stdint.h>
 
 namespace file {
+  void init();
+
   void const * open(char const * filename, uint32_t * out_size);
 
   void * openRelative(char const * filename, uint32_t * out_size);

src/file.cpp

@@ -1,3 +1,4 @@
+#include <assert.h>
 #include <stdio.h>
 #include <stdint.h>
 #include <stdlib.h>
@@ -7,6 +8,7 @@
 
 #include "pack.h"
 #include "file.h"
+#include "zlib.h"
 
 extern "C" {
 #ifdef __APPLE__
@@ -22,18 +24,41 @@ extern "C" {
 #endif
 };
 
+uint8_t * decompressed_start = NULL;
+
 namespace file {
 
+  void init()
+  {
+    uint32_t * header = (uint32_t *)files_pack_start;
+    if (header[0] != 0x56c8f1cb) {
+      fprintf(stderr, "invalid compressed header magic: %08x expected magic value: %08x\n", header[0], 0x56c8f1cb);
+      exit(EXIT_FAILURE);
+    }
+    printf("decompressed size %d\n", header[1]);
+    printf("compressed size %d\n", header[2]);
+
+    uint8_t * dest = (uint8_t *)malloc(header[1]);
+
+    uint64_t dest_len = header[1];
+    uint8_t * src = (uint8_t *)&header[3];
+    uint64_t src_len = header[2];
+    int ret = uncompress2(dest, &dest_len, src, &src_len);
+    assert(ret == Z_OK);
+    decompressed_start = dest;
+  }
+
   void const * open(const char * filename, uint32_t * out_size)
   {
+    assert(decompressed_start != NULL);
     fprintf(stderr, "(pack) filename: %s\n", filename);
 
-    pack::header const * header = (pack::header const *)&files_pack_start[0];
+    pack::header const * header = (pack::header const *)&decompressed_start[0];
     if (header->magic != pack::magic_value) {
       fprintf(stderr, "invalid header magic: %08x expected magic value: %08x\n", header->magic, pack::magic_value);
       exit(EXIT_FAILURE);
     }
-    ptrdiff_t data = (ptrdiff_t)&files_pack_start[header->header_size];
+    ptrdiff_t data = (ptrdiff_t)&decompressed_start[header->header_size];
 
     for (unsigned int i = 0; i < header->entry_count; i++) {
       if (strcmp(header->entry[i].filename, filename) == 0) {

src/main.cpp

@@ -355,6 +355,8 @@ void gamepad_update(view & viewState)
 
 int main()
 {
+  file::init();
+
   SDL_CHECK(SDL_Init(SDL_INIT_VIDEO | SDL_INIT_GAMEPAD));
   SDL_CHECK(SDL_Vulkan_LoadLibrary(NULL));
   volkInitialize();

src/pack.s

@@ -1,5 +1,6 @@
         .global _files_pack_start
         .global _files_pack_end
 _files_pack_start:
-        .incbin "files.pack"
+        //        .incbin "files.pack"
+        .incbin "files.pack.zlib"
 _files_pack_end:

src/renpy/interpreter.cpp

@@ -71,12 +71,17 @@ namespace renpy {
       fprintf(stderr, "interpret_one[%d]: scene_color\n", pc);
       backgroundIndex = -1;
       backgroundColor = statement.scene_color.color;
+
+      shownImagesCount = 0;
+      say.stringIndex = -1;
+      say.characterIndex = -1;
       pc += 1;
       break;
     case language::type::scene:
       fprintf(stderr, "interpret_one[%d]: scene\n", pc);
       assert(statement.scene.imageIndex < (uint32_t)script::images_length);
       backgroundIndex = statement.scene.imageIndex;
+
       shownImagesCount = 0;
       say.stringIndex = -1;
       say.characterIndex = -1;

tools/compress.c

@@ -0,0 +1,88 @@
+#include <assert.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "zlib.h"
+
+/*
+ZEXTERN int ZEXPORT compress2(Bytef *dest, uLongf *destLen,
+                              const Bytef *source, uLong sourceLen,
+                              int level);
+*/
+
+void const * read_file(const char * filename, int * out_size)
+{
+  fprintf(stderr, "filename: %s\n", filename);
+
+  FILE * f = fopen(filename, "rb");
+  if (f == NULL) {
+    fprintf(stderr, "fopen(%s): %s\n", filename, strerror(errno));
+    return NULL;
+  }
+
+  int fseek_end_ret = fseek(f, 0, SEEK_END);
+  if (fseek_end_ret < 0) {
+    fprintf(stderr, "fseek(%s, SEEK_END): %s\n", filename, strerror(errno));
+    return NULL;
+  }
+
+  size_t size = ftell(f);
+  if (size < 0) {
+    fprintf(stderr, "ftell(%s): %s\n", filename, strerror(errno));
+    return NULL;
+  }
+
+  int fseek_set_ret = fseek(f, 0, SEEK_SET);
+  if (fseek_set_ret < 0) {
+    fprintf(stderr, "lseek(%s, SEEK_SET): %s\n", filename, strerror(errno));
+    return NULL;
+  }
+  rewind(f);
+
+  void * buf = malloc(size);
+
+  size_t read_size = fread(buf, 1, size, f);
+  if (read_size != size) {
+    fprintf(stderr, "fread(%s): %s\n", filename, strerror(errno));
+    return NULL;
+  }
+
+  *out_size = size;
+
+  return buf;
+}
+
+int main(int argc, char const * const argv[])
+{
+  assert(argc == 3);
+  const char * in_filename = argv[1];
+  const char * out_filename = argv[2];
+
+  int out_size;
+  void const * buf = read_file(in_filename, &out_size);
+  assert(buf != NULL);
+
+  void * dest = malloc(out_size * 0.001 + out_size + 12);
+
+  static_assert((sizeof (long unsigned int)) == (sizeof (uint64_t)));
+  uint64_t dest_size = out_size;
+  int level = 9;
+  int zret = compress2(dest, &dest_size, buf, out_size, level);
+  assert(zret == Z_OK);
+
+  FILE * f = fopen(out_filename, "wb");
+  if (f == NULL) {
+    fprintf(stderr, "fopen(%s): %s\n", out_filename, strerror(errno));
+    return EXIT_FAILURE;
+  }
+
+  uint32_t header[3];
+  header[0] = 0x56c8f1cb; // magic
+  header[1] = out_size; // decompressed size
+  header[2] = dest_size; // compressed size
+  fwrite(header, (sizeof (uint32_t)), 3, f);
+  fwrite(dest, dest_size, 1, f);
+}

tools/compress.sh

@@ -0,0 +1,3 @@
+ZLIB=$HOME/zlib-1.3.2
+set -eux
+gcc -I$ZLIB compress.c $ZLIB/compress.o $ZLIB/deflate.o $ZLIB/trees.o $ZLIB/zutil.o $ZLIB/crc32.o $ZLIB/adler32.o -o compress