stencil outline

Makefile

@@ -24,7 +24,7 @@ CFLAGS += -I./include
 CFLAGS += -I../SDL3-dist/include
 CFLAGS += -fpic
 
-#FLAGS += -fstack-protector -fstack-protector-all -fno-omit-frame-pointer -fsanitize=address
+FLAGS += -fstack-protector -fstack-protector-all -fno-omit-frame-pointer -fsanitize=address
 
 LDFLAGS += -lm
 ifeq ($(UNAME),Linux)
@@ -63,15 +63,14 @@ all: main
 main: $(OBJS) $(LIBS) $(BINS) $(SHADERS)
 	$(CC) $(ARCH) $(LDFLAGS) $(FLAGS) $(OPT) $(DEBUG) $^ -o $@
 
-%.vs.spv: %.hlsl
-	../dxc/bin/dxc -spirv -T vs_6_1 -E VSMain -fspv-target-env=vulkan1.3 $< -Fo $@
-
-%.ps.spv: %.hlsl
-	../dxc/bin/dxc -spirv -T ps_6_1 -E PSMain -fspv-target-env=vulkan1.3 $< -Fo $@
+%.spv: %.hlsl
+	../dxc/bin/dxc -spirv -T lib_6_3 -fspv-target-env=vulkan1.3 $< -Fo $@
 
 tool/pack_file: tool/pack_file.cpp
 	make -C tool pack_file
 
+src/pack.o: files.pack
+
 PACK_FILENAMES = $(shell cat filenames.txt)
 files.pack: tool/pack_file $(PACK_FILENAMES) filenames.txt
 	./tool/pack_file $@ $(PACK_FILENAMES)

filenames.txt

@@ -1,5 +1,4 @@
-shader/triangle.vs.spv
-shader/triangle.ps.spv
+shader/triangle.spv
 index.idx
 position_normal_texture.vtx
 sprite.data

shader/triangle.hlsl

@@ -49,3 +49,25 @@ float4 PSMain(VSOutput input) : SV_TARGET
 
   return float4(color, 1.0);
 }
+
+struct VSOutlineOutput
+{
+  float4 Position : SV_POSITION;
+  float3 Normal : NORMAL0;
+};
+
+[shader("vertex")]
+VSOutlineOutput VSOutlineMain(VSInput input)
+{
+  VSOutlineOutput output = (VSOutlineOutput)0;
+  float3 position = input.Position.xyz + input.Normal.xyz * 0.01;
+  output.Position = mul(constants.data.Get().Projection, mul(constants.data.Get().View, mul(constants.data.Get().Model, float4(position, 1.0))));
+  output.Normal = input.Normal;
+  return output;
+}
+
+[shader("pixel")]
+float4 PSOutlineMain(VSOutlineOutput input) : SV_TARGET
+{
+  return float4(1.0, 1.0, 1.0, 1.0);
+}

src/main.cpp

@@ -109,7 +109,12 @@ VkSemaphore * renderSemaphores{ nullptr };
 VkCommandPool commandPool{ VK_NULL_HANDLE };
 VkCommandBuffer commandBuffers[maxFramesInFlight];
 
-VkPipeline pipeline{ VK_NULL_HANDLE };
+enum {
+  MAIN_PIPELINE = 0,
+  OUTLINE_PIPELINE = 1,
+};
+
+VkPipeline pipelines[2]{ VK_NULL_HANDLE, VK_NULL_HANDLE };
 VkPipelineLayout pipelineLayout{ VK_NULL_HANDLE };
 
 VkImage textureImage{ VK_NULL_HANDLE };
@@ -959,26 +964,16 @@ int main()
   // shaders
   //////////////////////////////////////////////////////////////////////
 
-  uint32_t triangleVSSize;
-  void const * triangleVSStart = file::open("shader/triangle.vs.spv", &triangleVSSize);
-  uint32_t trianglePSSize;
-  void const * trianglePSStart = file::open("shader/triangle.ps.spv", &trianglePSSize);
+  uint32_t triangleSize;
+  void const * triangleStart = file::open("shader/triangle.spv", &triangleSize);
 
-  VkShaderModuleCreateInfo vertexShaderModuleCreateInfo{
+  VkShaderModuleCreateInfo shaderModuleCreateInfo{
     .sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO,
-    .codeSize = triangleVSSize,
-    .pCode = (uint32_t *)triangleVSStart
+    .codeSize = triangleSize,
+    .pCode = (uint32_t *)triangleStart
   };
-  VkShaderModule vertexShaderModule{};
-  VK_CHECK(vkCreateShaderModule(device, &vertexShaderModuleCreateInfo, nullptr, &vertexShaderModule));
-
-  VkShaderModuleCreateInfo pixelShaderModuleCreateInfo{
-    .sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO,
-    .codeSize = trianglePSSize,
-    .pCode = (uint32_t *)trianglePSStart
-  };
-  VkShaderModule pixelShaderModule{};
-  VK_CHECK(vkCreateShaderModule(device, &pixelShaderModuleCreateInfo, nullptr, &pixelShaderModule));
+  VkShaderModule shaderModule{};
+  VK_CHECK(vkCreateShaderModule(device, &shaderModuleCreateInfo, nullptr, &shaderModule));
 
   //////////////////////////////////////////////////////////////////////
   // pipeline
@@ -1026,17 +1021,32 @@ int main()
     {
       .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
       .stage = VK_SHADER_STAGE_VERTEX_BIT,
-      .module = vertexShaderModule,
+      .module = shaderModule,
       .pName = "VSMain"
     },
     {
       .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
       .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
-      .module = pixelShaderModule,
+      .module = shaderModule,
       .pName = "PSMain"
     }
   };
 
+  VkPipelineShaderStageCreateInfo outlineShaderStages[2]{
+    {
+      .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
+      .stage = VK_SHADER_STAGE_VERTEX_BIT,
+      .module = shaderModule,
+      .pName = "VSOutlineMain"
+    },
+    {
+      .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
+      .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
+      .module = shaderModule,
+      .pName = "PSOutlineMain"
+    }
+  };
+
   VkPipelineViewportStateCreateInfo viewportState{
     .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
     .viewportCount = 1,
@@ -1071,11 +1081,29 @@ int main()
     },
   };
 
+  VkPipelineDepthStencilStateCreateInfo outlineDepthStencilState{
+    .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
+    .depthTestEnable = VK_TRUE,
+    .depthWriteEnable = VK_TRUE,
+    .depthCompareOp = VK_COMPARE_OP_LESS_OR_EQUAL,
+    .stencilTestEnable = VK_TRUE,
+    .front = {
+      .failOp = VK_STENCIL_OP_KEEP,
+      .passOp = VK_STENCIL_OP_REPLACE,
+      .depthFailOp = VK_STENCIL_OP_KEEP,
+      .compareOp = VK_COMPARE_OP_NOT_EQUAL,
+      .compareMask = 0x01,
+      .writeMask = 0x00,
+      .reference = 1,
+    },
+  };
+
   VkPipelineRenderingCreateInfo renderingCreateInfo{
     .sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO,
     .colorAttachmentCount = 1,
     .pColorAttachmentFormats = &surfaceFormat.format,
-    .depthAttachmentFormat = depthFormat
+    .depthAttachmentFormat = depthFormat,
+    .stencilAttachmentFormat = depthFormat
   };
 
   VkPipelineColorBlendAttachmentState blendAttachment{
@@ -1097,22 +1125,39 @@ int main()
     .rasterizationSamples = VK_SAMPLE_COUNT_1_BIT
   };
 
-  VkGraphicsPipelineCreateInfo pipelineCreateInfo{
-    .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
-    .pNext = &renderingCreateInfo,
-    .stageCount = 2,
-    .pStages = shaderStages,
-    .pVertexInputState = &vertexInputState,
-    .pInputAssemblyState = &inputAssemblyState,
-    .pViewportState = &viewportState,
-    .pRasterizationState = &rasterizationState,
-    .pMultisampleState = &multisampleState,
-    .pDepthStencilState = &depthStencilState,
-    .pColorBlendState = &colorBlendState,
-    .pDynamicState = &dynamicState,
-    .layout = pipelineLayout
+  VkGraphicsPipelineCreateInfo pipelineCreateInfos[2]{
+    {
+      .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
+      .pNext = &renderingCreateInfo,
+      .stageCount = 2,
+      .pStages = shaderStages,
+      .pVertexInputState = &vertexInputState,
+      .pInputAssemblyState = &inputAssemblyState,
+      .pViewportState = &viewportState,
+      .pRasterizationState = &rasterizationState,
+      .pMultisampleState = &multisampleState,
+      .pDepthStencilState = &depthStencilState,
+      .pColorBlendState = &colorBlendState,
+      .pDynamicState = &dynamicState,
+      .layout = pipelineLayout
+    },
+    {
+      .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
+      .pNext = &renderingCreateInfo,
+      .stageCount = 2,
+      .pStages = outlineShaderStages,
+      .pVertexInputState = &vertexInputState,
+      .pInputAssemblyState = &inputAssemblyState,
+      .pViewportState = &viewportState,
+      .pRasterizationState = &rasterizationState,
+      .pMultisampleState = &multisampleState,
+      .pDepthStencilState = &outlineDepthStencilState,
+      .pColorBlendState = &colorBlendState,
+      .pDynamicState = &dynamicState,
+      .layout = pipelineLayout
+    }
   };
-  VK_CHECK(vkCreateGraphicsPipelines(device, VK_NULL_HANDLE, 1, &pipelineCreateInfo, nullptr, &pipeline));
+  VK_CHECK(vkCreateGraphicsPipelines(device, VK_NULL_HANDLE, 2, pipelineCreateInfos, nullptr, pipelines));
 
   //////////////////////////////////////////////////////////////////////
   // loop
@@ -1248,7 +1293,6 @@ int main()
     VkRect2D scissor{ .extent{ .width = (uint32_t)windowSize.x, .height = (uint32_t)windowSize.y } };
     vkCmdSetScissor(commandBuffer, 0, 1, &scissor);
 
-    vkCmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
     VkDeviceSize vertexOffset{ 0 };
     vkCmdBindDescriptorSets(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, 0, 1, &textureDescriptorSet, 0, nullptr);
     vkCmdBindVertexBuffers(commandBuffer, 0, 1, &vertexIndexBuffer, &vertexOffset);
@@ -1257,7 +1301,12 @@ int main()
     vkCmdPushConstants(commandBuffer, pipelineLayout, VK_SHADER_STAGE_VERTEX_BIT, 0, (sizeof (VkDeviceAddress)), &shaderDataDevice.frame[frameIndex].deviceAddress);
     VkDeviceSize indexCount{ 9216 };
 
+    vkCmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelines[MAIN_PIPELINE]);
     vkCmdDrawIndexed(commandBuffer, indexCount, 3, 0, 0, 0);
+
+    vkCmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelines[OUTLINE_PIPELINE]);
+    vkCmdDrawIndexed(commandBuffer, indexCount, 3, 0, 0, 0);
+
     vkCmdEndRendering(commandBuffer);
 
     VkImageMemoryBarrier2 barrierPresent{
@@ -1348,12 +1397,12 @@ int main()
   vkDestroyDescriptorSetLayout(device, textureDescriptorSetLayout, nullptr);
   vkDestroyDescriptorPool(device, descriptorPool, nullptr);
   vkDestroyPipelineLayout(device, pipelineLayout, nullptr);
-  vkDestroyPipeline(device, pipeline, nullptr);
+  vkDestroyPipeline(device, pipelines[0], nullptr);
+  vkDestroyPipeline(device, pipelines[1], nullptr);
   vkDestroySwapchainKHR(device, swapchain, nullptr);
   vkDestroySurfaceKHR(instance, surface, nullptr);
   vkDestroyCommandPool(device, commandPool, nullptr);
-  vkDestroyShaderModule(device, vertexShaderModule, nullptr);
-  vkDestroyShaderModule(device, pixelShaderModule, nullptr);
+  vkDestroyShaderModule(device, shaderModule, nullptr);
 
   SDL_DestroyWindow(window);
   SDL_QuitSubSystem(SDL_INIT_VIDEO);

validation.sh

@@ -0,0 +1,2 @@
+export VK_INSTANCE_LAYERS=VK_LAYER_KHRONOS_validation
+export VK_VALIDATION_VALIDATE_SYNC=1