Gaussians CPU sorting (#2844)

Author: Meakk

application/F3DOptionsTools.cxx

@@ -183,7 +183,7 @@ static inline const std::array<CLIGroup, 8> CLIOptions = {{
       {"raytracing-denoise", "d", "Denoise the image", "<bool>", "1"} } },
 #endif
   {"PostFX (OpenGL)",
-    { {"blending", "p", R"(Select translucency blending mode ("none", "ddp", "sort" or "stochastic"))", "<string>", "ddp"},
+    { {"blending", "p", R"(Select translucency blending mode ("none", "ddp", "sort", "sort_cpu" or "stochastic"))", "<string>", "ddp"},
       {"translucency-support", "", "Enable translucency blending (deprecated)", "<bool>", "1"},
       {"ambient-occlusion", "q", "Enable ambient occlusion providing approximate shadows for better depth perception, implemented using SSAO", "<bool>", "1"},
       {"anti-aliasing", "a", R"(Select anti-aliasing method ("none", "fxaa", "ssaa" or "taa"))", "<string>", "fxaa"},

application/testing/tests.native.cmake

@@ -42,6 +42,7 @@ f3d_test(NAME TestVerboseQuakeMDLInvalid ARGS --verbose DATA w_medkit_hl.mdl REG
 
 if(VTK_VERSION VERSION_GREATER_EQUAL 9.3.20231102)
   f3d_test(NAME TestSPLAT DATA small.splat ARGS -osy --up=-Y --point-sprites-absolute-size --point-sprites-size=1)
+  f3d_test(NAME TestSPLATSortCPU DATA small.splat ARGS -osy --point-sprites=gaussian --up=-Y --point-sprites-absolute-size --point-sprites-size=1 --blending=sort_cpu --camera-position=2,0,0)
   f3d_test(NAME TestSPZ DATA hornedlizard_small_d0.spz ARGS -sy --point-sprites-absolute-size --point-sprites-size=1)
 
   set(_splat_args -sy --up=-Y --point-sprites-absolute-size --point-sprites-size=1 --camera-position=-2.00335,1.09654,-0.459485 --camera-focal-point=-0.796712,2.22795,0.705742)

doc/libf3d/03-OPTIONS.md

@@ -260,7 +260,7 @@ CLI: `--blending`.
 
 ### `render.effect.blending.mode` (_string_, default: `ddp`)
 
-Set the blending technique. Valid options are: `ddp` (dual depth peeling, quality), `sort` (only for gaussians), `stochastic` (fast)
+Set the blending technique. Valid options are: `ddp` (dual depth peeling, quality), `sort` (only for gaussians), `sort_cpu` (only for gaussians, slow), `stochastic` (fast)
 
 CLI: `--blending`.
 

doc/user/03-OPTIONS.md

@@ -497,12 +497,13 @@ _Denoise_ the image when using raytracing.
 ### `-p`, `--blending` (_string_, default: `ddp`)
 
 Enable _translucency blending support_.
-This is a technique used to correctly render translucent objects (`ddp`: dual depth peeling for quality, `sort`: for gaussians, `stochastic`: fast).
+This is a technique used to correctly render translucent objects (`ddp`: dual depth peeling for quality, `sort`: for gaussians, `sort_cpu`: for gaussians, `stochastic`: fast).
 
 > [!WARNING]
 > `stochastic` is introducing a lot of noise with strong translucency.
 > It works better when combined with temporal anti-aliasing (when using `--anti-aliasing=taa` option)
 > `sort` is only working for 3D gaussians and requires compute shaders support.
+> Alternatively, `sort_cpu` will give the same result and work everywhere but it's much slower.
 
 ### `-q`, `--ambient-occlusion` (_bool_, default: `false`)
 

library/src/interactor_impl.cxx

@@ -831,7 +831,7 @@ interactor& interactor_impl::initCommands()
   static const std::map<std::string, std::vector<std::string>> COMPL_OPTIONS_SET = {
     { "model.point_sprites.type", { "sphere", "gaussian" } },
     { "render.effect.antialiasing.mode", { "fxaa", "ssaa", "taa" } },
-    { "render.effect.blending.mode", { "ddp", "sort", "stochastic" } },
+    { "render.effect.blending.mode", { "ddp", "sort", "sort_cpu", "stochastic" } },
   };
   auto complOptionSet = [&](const std::vector<std::string>& args)
   {
@@ -1006,6 +1006,10 @@ interactor& interactor_impl::initCommands()
           mode = "sort";
         }
         else if (mode == "sort")
+        {
+          mode = "sort_cpu";
+        }
+        else if (mode == "sort_cpu")
         {
           mode = "stochastic";
         }
@@ -1017,7 +1021,7 @@ interactor& interactor_impl::initCommands()
       this->Internals->Window.render();
     },
     command_documentation_t{
-      "cycle_blending", "cycle between the blending method (none,ddp,sort,stochastic)" });
+      "cycle_blending", "cycle between the blending method (none,ddp,sort,sort_cpu,stochastic)" });
 
   std::vector<std::string> cycleColoringValidArgs = { "field", "array", "component" };
   this->addCommand(

library/src/window_impl.cxx

@@ -438,7 +438,8 @@ void window_impl::UpdateDynamicOptions()
     renderer->SetPointSpritesType(splatType);
     renderer->SetPointSpritesSize(
       opt.model.point_sprites.absolute_size, opt.model.point_sprites.size);
-    renderer->SetPointSpritesUseInstancing(opt.render.effect.blending.mode != "sort");
+    renderer->SetPointSpritesUseInstancing(
+      opt.render.effect.blending.mode != "sort" && opt.render.effect.blending.mode != "sort_cpu");
   }
 
   renderer->SetLineWidth(opt.render.line_width);
@@ -570,14 +571,18 @@ void window_impl::UpdateDynamicOptions()
     {
       blendMode = vtkF3DRenderer::BlendingMode::SORT;
     }
+    else if (opt.render.effect.blending.mode == "sort_cpu")
+    {
+      blendMode = vtkF3DRenderer::BlendingMode::SORT_CPU;
+    }
     else if (opt.render.effect.blending.mode == "stochastic")
     {
       blendMode = vtkF3DRenderer::BlendingMode::STOCHASTIC;
     }
     else
     {
       log::warn(opt.render.effect.blending.mode,
-        R"( is an invalid blending mode. Valid modes are: "ddp", "sort", "stochastic")");
+        R"( is an invalid blending mode. Valid modes are: "ddp", "sort", "sort_cpu", "stochastic")");
     }
   }
 

testing/baselines/TestSPLATSortCPU.png

@@ -13,6 +13,11 @@ KeyPressEvent -465 475 0 112 1 p 0
 CharEvent -465 475 0 112 1 p 0
 KeyReleaseEvent -465 475 0 112 1 p 0
 
+# P
+KeyPressEvent -465 475 0 112 1 p 0
+CharEvent -465 475 0 112 1 p 0
+KeyReleaseEvent -465 475 0 112 1 p 0
+
 # Right
 KeyPressEvent -465 475 0 0 1 Right 0
 CharEvent -465 475 0 0 1 Right 0

testing/recordings/TestInteractionAndCLIBlending.log

@@ -16,3 +16,6 @@ KeyReleaseEvent 589 299 0 118 1 p
 KeyPressEvent 589 299 0 118 1 p
 CharEvent 589 299 0 118 1 p
 KeyReleaseEvent 589 299 0 118 1 p
+KeyPressEvent 589 299 0 118 1 p
+CharEvent 589 299 0 118 1 p
+KeyReleaseEvent 589 299 0 118 1 p

testing/recordings/TestInteractionCycleBlending.log

@@ -34,7 +34,9 @@
 #include <vtk_glew.h>
 #endif
 
+#include <algorithm>
 #include <sstream>
+#include <vector>
 
 //----------------------------------------------------------------------------
 class vtkF3DSplatMapperHelper : public vtkOpenGLPointGaussianMapperHelper
@@ -76,17 +78,22 @@ class vtkF3DSplatMapperHelper : public vtkOpenGLPointGaussianMapperHelper
 
 private:
 #if !defined(__ANDROID__) && !defined(__EMSCRIPTEN__)
-  void SortSplats(vtkRenderer* ren);
-
   vtkNew<vtkShader> DepthComputeShader;
   vtkNew<vtkShaderProgram> DepthProgram;
   vtkNew<vtkOpenGLBufferObject> DepthBuffer;
 
   vtkNew<vtkF3DBitonicSort> Sorter;
+#endif
+
+  std::vector<unsigned int> CPUSortedIndices;
+  std::vector<float> CPUDepths;
 
-  double DirectionThreshold = 0.999;
+  static constexpr double DirectionThreshold = 0.999;
   double LastDirection[3] = { 0.0, 0.0, 0.0 };
-#endif
+
+  bool SortNeeded(vtkRenderer* ren);
+  void SortSplats(vtkRenderer* ren);
+  void SortSplatsCPU(vtkRenderer* ren);
 
   bool OwnerUseInstancing();
 
@@ -311,71 +318,129 @@ void vtkF3DSplatMapperHelper::SetCameraShaderParameters(
   this->Superclass::SetCameraShaderParameters(cellBO, ren, actor);
 }
 
-#if !defined(__ANDROID__) && !defined(__EMSCRIPTEN__)
+//------------------------------------------------------------------------------
+bool vtkF3DSplatMapperHelper::SortNeeded(vtkRenderer* ren)
+{
+  const double* focalPoint = ren->GetActiveCamera()->GetFocalPoint();
+  const double* origin = ren->GetActiveCamera()->GetPosition();
+  double direction[3];
+
+  for (int i = 0; i < 3; ++i)
+  {
+    // the orientation is reverted to sort splats back to front
+    direction[i] = origin[i] - focalPoint[i];
+  }
+
+  vtkMath::Normalize(direction);
+
+  if (vtkMath::Dot(this->LastDirection, direction) >= vtkF3DSplatMapperHelper::DirectionThreshold)
+  {
+    return false;
+  }
+
+  this->LastDirection[0] = direction[0];
+  this->LastDirection[1] = direction[1];
+  this->LastDirection[2] = direction[2];
+
+  return true;
+}
+
 //----------------------------------------------------------------------------
 void vtkF3DSplatMapperHelper::SortSplats(vtkRenderer* ren)
 {
-  int numVerts = this->VBOs->GetNumberOfTuples("vertexMC");
+#if !defined(__ANDROID__) && !defined(__EMSCRIPTEN__)
 
-  if (numVerts)
+  if (!this->SortNeeded(ren))
   {
-    const double* focalPoint = ren->GetActiveCamera()->GetFocalPoint();
-    const double* origin = ren->GetActiveCamera()->GetPosition();
-    double direction[3];
+    return;
+  }
 
-    for (int i = 0; i < 3; ++i)
-    {
-      // the orientation is reverted to sort splats back to front
-      direction[i] = origin[i] - focalPoint[i];
-    }
+  int numVerts = this->VBOs->GetNumberOfTuples("vertexMC");
 
-    vtkMath::Normalize(direction);
+  vtkOpenGLShaderCache* shaderCache =
+    vtkOpenGLRenderWindow::SafeDownCast(ren->GetRenderWindow())->GetShaderCache();
 
-    // sort the splats only if the camera direction has changed
-    if (vtkMath::Dot(this->LastDirection, direction) < this->DirectionThreshold)
-    {
-      vtkOpenGLShaderCache* shaderCache =
-        vtkOpenGLRenderWindow::SafeDownCast(ren->GetRenderWindow())->GetShaderCache();
+  // compute next power of two
+  unsigned int numVertsExt = vtkMath::NearestPowerOfTwo(numVerts);
 
-      // compute next power of two
-      unsigned int numVertsExt = vtkMath::NearestPowerOfTwo(numVerts);
+  // depth computation
+  shaderCache->ReadyShaderProgram(this->DepthProgram);
 
-      // depth computation
-      shaderCache->ReadyShaderProgram(this->DepthProgram);
+  this->DepthProgram->SetUniform3f("viewDirection", this->LastDirection);
+  this->DepthProgram->SetUniformi("count", numVerts);
+  this->VBOs->GetVBO("vertexMC")->BindShaderStorage(0);
+  this->Primitives[PrimitivePoints].IBO->BindShaderStorage(1);
+  this->DepthBuffer->BindShaderStorage(2);
 
-      this->LastDirection[0] = direction[0];
-      this->LastDirection[1] = direction[1];
-      this->LastDirection[2] = direction[2];
+  glDispatchCompute(numVertsExt / 32, 1, 1);
+  glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT);
 
-      this->DepthProgram->SetUniform3f("viewDirection", direction);
-      this->DepthProgram->SetUniformi("count", numVerts);
-      this->VBOs->GetVBO("vertexMC")->BindShaderStorage(0);
-      this->Primitives[PrimitivePoints].IBO->BindShaderStorage(1);
-      this->DepthBuffer->BindShaderStorage(2);
+  // sort
+  this->Sorter->Run(vtkOpenGLRenderWindow::SafeDownCast(ren->GetRenderWindow()), numVerts,
+    this->DepthBuffer, this->Primitives[PrimitivePoints].IBO);
+#endif
+}
 
-      glDispatchCompute(numVertsExt / 32, 1, 1);
-      glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT);
+//----------------------------------------------------------------------------
+void vtkF3DSplatMapperHelper::SortSplatsCPU(vtkRenderer* ren)
+{
+  if (!this->SortNeeded(ren))
+  {
+    return;
+  }
 
-      // sort
-      this->Sorter->Run(vtkOpenGLRenderWindow::SafeDownCast(ren->GetRenderWindow()), numVerts,
-        this->DepthBuffer, this->Primitives[PrimitivePoints].IBO);
-    }
+  int numVerts = this->VBOs->GetNumberOfTuples("vertexMC");
+
+  this->CPUSortedIndices.resize(static_cast<size_t>(numVerts));
+
+  this->Primitives[PrimitivePoints].IBO->Download(
+    this->CPUSortedIndices.data(), static_cast<size_t>(numVerts));
+
+  this->CPUDepths.resize(static_cast<size_t>(numVerts));
+  // compute depth for each splat
+  vtkPolyData* poly = this->CurrentInput;
+
+  vtkPoints* points = poly->GetPoints();
+
+  for (int i = 0; i < numVerts; ++i)
+  {
+    const double* pos = points->GetPoint(i);
+    this->CPUDepths[i] = pos[0] * this->LastDirection[0] + pos[1] * this->LastDirection[1] +
+      pos[2] * this->LastDirection[2];
   }
+
+  // Match bitonic sort ordering: sort ascending by depth (back-to-front given reversed direction)
+  std::sort(this->CPUSortedIndices.begin(), this->CPUSortedIndices.end(),
+    [&](const GLuint& a, const GLuint& b) { return this->CPUDepths[a] < this->CPUDepths[b]; });
+
+  this->Primitives[PrimitivePoints].IBO->Upload(this->CPUSortedIndices.data(),
+    static_cast<size_t>(numVerts), vtkOpenGLBufferObject::ObjectType::ElementArrayBuffer);
 }
-#endif
 
 //----------------------------------------------------------------------------
 void vtkF3DSplatMapperHelper::RenderPieceDraw(vtkRenderer* ren, vtkActor* actor)
 {
-#if !defined(__ANDROID__) && !defined(__EMSCRIPTEN__)
   const vtkF3DRenderer* renderer = vtkF3DRenderer::SafeDownCast(ren);
 
-  if (renderer->GetBlendingMode() == vtkF3DRenderer::BlendingMode::SORT &&
-    vtkShader::IsComputeShaderSupported() && actor->HasTranslucentPolygonalGeometry())
+  if (actor->HasTranslucentPolygonalGeometry())
   {
-    this->SortSplats(ren);
+    if (renderer->GetBlendingMode() == vtkF3DRenderer::BlendingMode::SORT)
+    {
+      if (vtkShader::IsComputeShaderSupported())
+      {
+        this->SortSplats(ren);
+      }
+      else
+      {
+        vtkWarningMacro("Compute shaders not supported, falling back to CPU sorting");
+        this->SortSplatsCPU(ren);
+      }
+    }
+    else if (renderer->GetBlendingMode() == vtkF3DRenderer::BlendingMode::SORT_CPU)
+    {
+      this->SortSplatsCPU(ren);
+    }
   }
-#endif
 
   if (this->OwnerUseInstancing())
   {