Add CUDA toggle scaffolding

Author: AhmedSalih3d

README.md

@@ -90,6 +90,8 @@ To color exported cell grids by particle counts, set
 one plus the particles in its neighbor stencil.
 Neighbor rebuilds use a per-cell ordering buffer without reordering particle
 storage.
+To toggle the CUDA path (requires CUDA.jl), call `EnableCUDA!(SimMetaData)` or
+set `SimMetaData.CUDASettings.Enabled = true` before running a simulation.
 
 ## Help
 

example/StillWedgeMDBC.jl

@@ -7,7 +7,7 @@ let
     SimConstantsWedge = SimulationConstants{FloatType}(dx=0.02,c₀=42.48576250492629, δᵩ = 0.1, CFL=0.5)
     # SimConstantsWedge = SimulationConstants{FloatType}(dx=0.01,c₀=43.4, δᵩ = 0.1, CFL=0.2)
 # 
-    SimMetaDataWedge  = SimulationMetaData{Dimensions,FloatType,NoShifting,NoKernelOutput,NoMDBC,StoreLog}(
+    SimMetaDataWedge  = SimulationMetaData{Dimensions,FloatType,NoShifting,NoKernelOutput,SimpleMDBC,StoreLog}(
         SimulationName="StillWedge", 
         SaveLocation="W:/Simulations/StillWedge2D_MDBC",
         SimulationTime=4.0,
@@ -18,6 +18,8 @@ let
         OpenLogFile=true,
     )
 
+    EnableCUDA!(SimMetaDataWedge)
+
     # If save directory is not already made, make it
     if !isdir(SimMetaDataWedge.SaveLocation)
         mkdir(SimMetaDataWedge.SaveLocation)

src/SPHCellList.jl

@@ -9,6 +9,7 @@ using ..SimulationEquations
 using ..SimulationGeometry
 using ..AuxiliaryFunctions
 using ..SimulationMetaDataConfiguration
+using ..SimulationCUDAConfiguration: IsCUDAEnabled
 using ..SimulationConstantsConfiguration
 using ..SimulationLoggerConfiguration
 using ..PreProcess
@@ -19,6 +20,7 @@ using ..SPHKernels
 using ..SPHViscosityModels
 using ..SPHDensityDiffusionModels
 using ..SPHNeighborList: BuildNeighborCellLists!, ComputeCellNeighborCounts, ComputeCellParticleCounts, ConstructStencil, ExtractCells!, MapFloor, UpdateNeighbors!, UpdateΔx!
+using ..SPHNeighborListCUDA: UpdateNeighborsCUDA!
 
 using StaticArrays
 using StructArrays: StructArray, foreachfield
@@ -747,7 +749,11 @@ using LinearAlgebra
                     # Remove if statement logic if you want to update each iteration
                     # if mod(SimMetaData.Iteration, ceil(Int, SimKernel.H / (SimConstants.c₀ * dt * (1/SimConstants.CFL)) )) == 0 || SimMetaData.Iteration == 1
                     if ShouldRebuild
-                        @timeit SimMetaData.HourGlass "01a Actual Calculate IndexCounter" SimMetaData.IndexCounter = UpdateNeighbors!(SimParticles, SimKernel.H⁻¹, ParticleRanges, UniqueCells, CellDict, ParticleOrder, CellOffsets)
+                        if IsCUDAEnabled(SimMetaData)
+                            @timeit SimMetaData.HourGlass "01a Actual Calculate IndexCounter" SimMetaData.IndexCounter = UpdateNeighborsCUDA!(SimParticles, SimKernel.H⁻¹, ParticleRanges, UniqueCells, CellDict, ParticleOrder, CellOffsets)
+                        else
+                            @timeit SimMetaData.HourGlass "01a Actual Calculate IndexCounter" SimMetaData.IndexCounter = UpdateNeighbors!(SimParticles, SimKernel.H⁻¹, ParticleRanges, UniqueCells, CellDict, ParticleOrder, CellOffsets)
+                        end
                         SimMetaData.Δx    = zero(eltype(dρdtI))
                         UniqueCellsView   = view(UniqueCells, 1:SimMetaData.IndexCounter)
                         BuildNeighborCellLists!(NeighborCellLists, FullStencil, UniqueCellsView, ParticleRanges, CellDict)

src/SPHExample.jl

@@ -5,6 +5,7 @@ module SPHExample
     include("SPHViscosityModels.jl")      
     include("ProduceHDFVTK.jl")    
     include("SimulationGeometry.jl")
+    include("SimulationCUDAConfiguration.jl");
     include("SimulationMetaDataConfiguration.jl");
     include("SimulationEquations.jl");
     include("TimeStepping.jl");       
@@ -14,6 +15,7 @@ module SPHExample
     include("OpenExternalPrograms.jl")
     include("SPHDensityDiffusionModels.jl")  
     include("SPHNeighborList.jl")
+    include("SPHNeighborListCUDA.jl")
     include("SPHCellList.jl") #Must be last    
 
     # Re-export desired functions from each submodule
@@ -51,7 +53,11 @@ module SPHExample
     export SimulationMetaData, ShiftingMode, NoShifting, PlanarShifting,
            KernelOutputMode, NoKernelOutput, StoreKernelOutput,
            MDBCMode, NoMDBC, SimpleMDBC,
-           LogMode, NoLog, StoreLog
+           LogMode, NoLog, StoreLog,
+           CUDASettings, EnableCUDA!, DisableCUDA!, IsCUDAEnabled
+
+    using .SimulationCUDAConfiguration
+    export CUDASettings, EnableCUDA!, DisableCUDA!, IsCUDAEnabled
 
     using .SimulationConstantsConfiguration
     export SimulationConstants
@@ -60,6 +66,9 @@ module SPHExample
     export ConstructStencil, ExtractCells!, UpdateNeighbors!,
            BuildNeighborCellLists!, ComputeCellParticleCounts, ComputeCellNeighborCounts
 
+    using .SPHNeighborListCUDA
+    export UpdateNeighborsCUDA!
+
     using .SPHCellList
     export NeighborLoop!, ComputeInteractions!, RunSimulation
 

src/SPHNeighborListCUDA.jl

@@ -0,0 +1,15 @@
+module SPHNeighborListCUDA
+
+export UpdateNeighborsCUDA!
+
+using ..SPHNeighborList: UpdateNeighbors!
+using ..SimulationCUDAConfiguration: AssertCUDAAvailable
+
+function UpdateNeighborsCUDA!(Particles, InverseCutOff, ParticleRanges,
+                              UniqueCells, CellDict, ParticleOrder, CellOffsets)
+    AssertCUDAAvailable()
+    return UpdateNeighbors!(Particles, InverseCutOff, ParticleRanges,
+                            UniqueCells, CellDict, ParticleOrder, CellOffsets)
+end
+
+end

src/SimulationCUDAConfiguration.jl

@@ -0,0 +1,31 @@
+module SimulationCUDAConfiguration
+
+export CUDASettings, EnableCUDA!, DisableCUDA!, IsCUDAEnabled, AssertCUDAAvailable
+
+using Parameters
+
+@with_kw mutable struct CUDASettings
+    Enabled::Bool = false
+end
+
+function AssertCUDAAvailable()
+    if Base.find_package("CUDA") === nothing
+        error("CUDA.jl not available. Add CUDA.jl to Project.toml and instantiate.")
+    end
+    return nothing
+end
+
+function EnableCUDA!(Settings::CUDASettings)
+    Settings.Enabled = true
+    AssertCUDAAvailable()
+    return nothing
+end
+
+function DisableCUDA!(Settings::CUDASettings)
+    Settings.Enabled = false
+    return nothing
+end
+
+IsCUDAEnabled(Settings::CUDASettings) = Settings.Enabled
+
+end

src/SimulationMetaDataConfiguration.jl

@@ -1,13 +1,17 @@
 module SimulationMetaDataConfiguration
 
 using Parameters
+
+using ..SimulationCUDAConfiguration: CUDASettings, EnableCUDA!, DisableCUDA!, IsCUDAEnabled
 using TimerOutputs
 
+import ..SimulationCUDAConfiguration: EnableCUDA!, DisableCUDA!, IsCUDAEnabled
 
 export SimulationMetaData, UpdateMetaData!, ShiftingMode, NoShifting, PlanarShifting,
        KernelOutputMode, NoKernelOutput, StoreKernelOutput,
        MDBCMode, NoMDBC, SimpleMDBC,
-       LogMode, NoLog, StoreLog
+       LogMode, NoLog, StoreLog,
+       CUDASettings, EnableCUDA!, DisableCUDA!, IsCUDAEnabled
 
 abstract type ShiftingMode end
 struct NoShifting    <: ShiftingMode end
@@ -48,6 +52,7 @@ struct StoreLog <: LogMode end
     ExportSingleVTKHDF::Bool                = true
     ExportGridCells::Bool                   = false
     ExportGridCellParticleCounts::Bool      = false
+    CUDASettings::CUDASettings              = CUDASettings()
     OutputVariables::Vector{String}         = [
         "Density",
         "Pressure",
@@ -59,6 +64,10 @@ struct StoreLog <: LogMode end
     OpenLogFile::Bool                       = true
     Δx::FloatType                           = zero(FloatType)
 end
+
+EnableCUDA!(SimMetaData::SimulationMetaData) = EnableCUDA!(SimMetaData.CUDASettings)
+DisableCUDA!(SimMetaData::SimulationMetaData) = DisableCUDA!(SimMetaData.CUDASettings)
+IsCUDAEnabled(SimMetaData::SimulationMetaData) = IsCUDAEnabled(SimMetaData.CUDASettings)
 SimulationMetaData{D,T,S,K,B}(; kwargs...) where {D,T,S<:ShiftingMode,K<:KernelOutputMode,B<:MDBCMode} =
     SimulationMetaData{D,T,S,K,B,NoLog}(; kwargs...)
 SimulationMetaData{D,T,S,K}(; kwargs...) where {D,T,S<:ShiftingMode,K<:KernelOutputMode} =