From ec525bbfbc21dd920c4c74ed94192aa241e3f663 Mon Sep 17 00:00:00 2001
From: Zack Buhman <zack@buhman.org>
Date: Sat, 11 Jan 2025 01:46:42 -0600
Subject: [PATCH] execute: implement instanceof for arrays of references

---
 Makefile                         |  2 +-
 c/class_resolver.c               | 40 +++++++++++++++++++++++++++++++-
 c/execute.c                      | 25 +++++++++++++++++---
 classes/test/TestInstanceof.java | 11 +++++++++
 4 files changed, 73 insertions(+), 5 deletions(-)
 create mode 100644 classes/test/TestInstanceof.java

diff --git a/Makefile b/Makefile
index e5f3bf1..5704539 100644
--- a/Makefile
+++ b/Makefile
@@ -10,7 +10,7 @@ CFLAGS += -Wall -Werror -Wfatal-errors -Wno-error=unused-variable -fstack-protec
 CFLAGS += -I$(MAKEFILE_PATH)/
 CFLAGS += -I$(MAKEFILE_PATH)/c
 CFLAGS += -DDEBUG
-#CFLAGS += -DDEBUG_PRINT
+CFLAGS += -DDEBUG_PRINT
 LDFLAGS = -lm
 OPT ?= -O0
 DEPFLAGS = -MMD -MP
diff --git a/c/class_resolver.c b/c/class_resolver.c
index be17819..1822134 100644
--- a/c/class_resolver.c
+++ b/c/class_resolver.c
@@ -628,6 +628,13 @@ bool class_resolver_instanceof(int class_hash_table_length,
                                const int class_index,
                                struct objectref * objectref)
 {
+  struct constant * class_constant = &origin_class_entry->class_file->constant_pool[class_index - 1];
+  assert(class_constant->tag == CONSTANT_Class);
+  struct constant * class_name_constant =  &origin_class_entry->class_file->constant_pool[class_constant->class.name_index - 1];
+  assert(class_name_constant->tag == CONSTANT_Utf8);
+
+  int class_name_ix = 0;
+
   struct class_entry * index_class_entry =
     class_resolver_lookup_class_from_class_index(class_hash_table_length,
                                                  class_hash_table,
@@ -636,7 +643,38 @@ bool class_resolver_instanceof(int class_hash_table_length,
   assert(index_class_entry != nullptr);
 
   assert(objectref != nullptr);
-  struct class_entry * class_entry = objectref->class_entry;
+
+  struct class_entry * class_entry;
+
+  while (true) {
+    struct tag * tag = (struct tag *)objectref;
+    if (tag->type == TAG_TYPE_REF_ARRAY || tag->type == TAG_TYPE_PRIM_ARRAY) {
+      if (class_name_constant->utf8.bytes[class_name_ix] != '[')
+        return false;
+
+      class_name_ix += 1;
+
+      struct arrayref * arrayref = (struct arrayref *)objectref;
+      if (arrayref->class_entry != nullptr) {
+        class_entry = arrayref->class_entry;
+        break;
+      } else {
+        // FIXME: instanceof is allowed on zero-length arrays
+        assert(arrayref->length > 0);
+        objectref = arrayref->oref[0];
+      }
+    } else if (tag->type == TAG_TYPE_OBJECT) {
+      class_entry = objectref->class_entry;
+      break;
+    } else {
+      assert(!"invalid tag type");
+    }
+  }
+
+  assert(class_entry != nullptr);
+  if (class_name_constant->utf8.bytes[class_name_ix] == '[')
+    return false;
+
   while (true) {
     debugf("class_entry: %p\n", class_entry);
 
diff --git a/c/execute.c b/c/execute.c
index a91d193..107d883 100644
--- a/c/execute.c
+++ b/c/execute.c
@@ -1745,15 +1745,34 @@ static struct arrayref * _multiarray(struct vm * vm, int32_t * dims, int num_dim
   int32_t count = dims[level];
   struct arrayref * arrayref;
   int32_t element_size;
-  if (*type == 'L' || *type == '[') {
+  if (*type == '[') {
     element_size = (sizeof (void *));
     arrayref = ref_array_allocate(vm, count);
+    assert(arrayref != nullptr);
+    arrayref->class_entry = nullptr;
+  } else if (*type == 'L') {
+    element_size = (sizeof (void *));
+    arrayref = ref_array_allocate(vm, count);
+    assert(arrayref != nullptr);
+
+    int type_length = type_end - type;
+
+    struct parse_type_ret parse_type_ret = parse_type(type,
+                                                      type_length);
+
+    struct class_entry * class_entry = class_resolver_lookup_class(vm->class_hash_table.length,
+                                                                   vm->class_hash_table.entry,
+                                                                   parse_type_ret.bytes,
+                                                                   parse_type_ret.length);
+    assert(class_entry != nullptr);
+
+    arrayref->class_entry = class_entry;
   } else {
     element_size = field_size_array(*type);
     arrayref = prim_array_allocate(vm, element_size, count);
+    assert(arrayref != nullptr);
+    arrayref->class_entry = nullptr;
   }
-  assert(arrayref != nullptr);
-  arrayref->class_entry = nullptr;
 
   int32_t array_element_size = count * element_size; // bytes
   int32_t u32_count = (array_element_size + 3) / 4;  // u32 units
diff --git a/classes/test/TestInstanceof.java b/classes/test/TestInstanceof.java
new file mode 100644
index 0000000..b74f813
--- /dev/null
+++ b/classes/test/TestInstanceof.java
@@ -0,0 +1,11 @@
+package test;
+
+class TestInstanceof {
+    static boolean test1() {
+        Object[][] a = new Object[1][1];
+        return a instanceof Object[][];
+    }
+    public static void main() {
+        System.out.println(test1());
+    }
+}