Tessellation: Hull & Domain Shaders

Tessellation subdivides patches on the GPU: Hull shader (per control point + a patch-constant function computing tessellation factors) → fixed-function tessellator → Domain shader (positions each generated vertex).

Tessellation subdivides patches on the GPU: Hull shader (per control point + a patch-constant function computing tessellation factors) → fixed-function tessellator → Domain shader (positions each generated vertex). Topology must be a patch list (D3D_PRIMITIVE_TOPOLOGY_3_CONTROL_POINT_PATCHLIST, PSO topology type PATCH).

26.1 Hull shader

Setting any edge factor to 0 culls the patch — useful for frustum/back-face culling in the patch constant function.

struct VSToHS { float3 posW : POSITION; float3 normalW : NORMAL; float2 uv : TEXCOORD0; };
struct HSOut  { float3 posW : POSITION; float3 normalW : NORMAL; float2 uv : TEXCOORD0; };
struct PatchTess
{
    float edges[3] : SV_TessFactor;
    float inside   : SV_InsideTessFactor;
};

cbuffer TessCB : register(b0)
{
    float4x4 gViewProjT;
    float3   gEyePos;
    float    gMaxTess;
    float    gDispScale;
};

float TessFactorForEdge(float3 a, float3 b)
{
    float d = distance((a + b) * 0.5, gEyePos);
    return clamp(gMaxTess * saturate(1.0 - d / 100.0), 1.0, gMaxTess);   // distance-based LOD
}

PatchTess PatchConstantFn(InputPatch<VSToHS, 3> patch, uint pid : SV_PrimitiveID)
{
    PatchTess pt;
    pt.edges[0] = TessFactorForEdge(patch[1].posW, patch[2].posW);   // edge opposite vertex 0
    pt.edges[1] = TessFactorForEdge(patch[2].posW, patch[0].posW);
    pt.edges[2] = TessFactorForEdge(patch[0].posW, patch[1].posW);
    pt.inside   = (pt.edges[0] + pt.edges[1] + pt.edges[2]) / 3.0;
    return pt;
}

[domain("tri")]
[partitioning("fractional_odd")]        // integer, fractional_even, fractional_odd, pow2
[outputtopology("triangle_cw")]         // point, line, triangle_cw, tri
…

26.2 Domain shader with displacement

Domains: tri (barycentric float3), quad (float2 UV, 4 edge + 2 inside factors), isoline. Techniques: PN triangles, Phong tessellation, terrain, displacement mapping. Use precise on position math to avoid cracks between patches. On modern hardware mesh shaders or compute-based subdivision often replace hardware tessellation.

Texture2D<float> gHeightMap : register(t0);
SamplerState     gLinearClamp : register(s0);
struct DSOut { float4 pos : SV_Position; float3 normalW : NORMAL; float2 uv : TEXCOORD0; };

[domain("tri")]
DSOut DSMain(PatchTess pt, float3 bary : SV_DomainLocation, const OutputPatch<HSOut, 3> patch)
{
    DSOut o;
    float3 p = bary.x * patch[0].posW + bary.y * patch[1].posW + bary.z * patch[2].posW;
    float3 n = normalize(bary.x * patch[0].normalW + bary.y * patch[1].normalW + bary.z * patch[2].normalW);
    float2 uv = bary.x * patch[0].uv + bary.y * patch[1].uv + bary.z * patch[2].uv;
    float h = gHeightMap.SampleLevel(gLinearClamp, uv, 0);       // no derivatives in DS
    p += n * h * gDispScale;
    o.pos = mul(gViewProjT, float4(p, 1));
    o.normalW = n; o.uv = uv;
    return o;
}