2d Cel Shading using DirectX

216.239.59.104, where else?

Also, I decided to mess around with HLSL using EffectEdit from the SDK, and came up with this:

string XFile = "teapot.x";
int    BCLR  = 0xFF0080FF;

// transformations

float4x3 WorldView  : WORLDVIEW;
float4x4 Projection : PROJECTION;

// light direction (view space)

float3 lightDir <  string UIDirectional = "Light Direction"; > = {0.577, -0.577, 0.577};

// light intensity

float4 I_a = { 0.3f, 0.3f, 0.3f, 1.0f };    // ambient

float4 I_d = { 1.0f, 1.0f, 1.0f, 1.0f };    // diffuse

float4 I_s = { 0.6f, 0.6f, 0.6f, 1.0f };    // specular


// material reflectivity

float4 k_a = { 0.2f, 0.2f, 0.2f, 1.0f };    // ambient

float4 k_d = { 1.0f, 0.7f, 0.2f, 1.0f };    // diffuse

float4 k_s = { 1.0f, 1.0f, 1.0f, 1.0f };    // specular

float  n   = 64.0f;                         // power


// textures

texture Tex < string name = "caust31.tga"; >;
texture Tex2 < string function = "temp"; int width = 16; int height = 16; >;

float4 temp(float2 Pos : POSITION) : COLOR
{
    float4 output;

    float4 ints;

    ints.x = (sin(Pos.x * 2.0f) + 1.0f) / 2.0f;
    ints.y = (sin(Pos.y * 5.0f) + 1.0f) / 2.0f;
    ints.z = (cos(Pos.x * 2.0f) + 1.0f) / 2.0f;
    ints.w = (tan(Pos.y * 25.0f) + 1.0f) / 2.0f;

    output.x = (ints.x * 0.1f + ints.y * 0.3f + ints.z + 0.6f) * ints.w * ints.x;
    output.yzw = 0.0f;

    return output;    
}

struct VS_OUTPUT
{
    float4 Pos  : POSITION;
    float3 Diff : COLOR0;
    float3 Spec : COLOR1;
    float2 Tex0 : TEXCOORD0;               
};

struct PS_OUTPUT
{
    float4 col;
};

VS_OUTPUT VS(float3 Pos : POSITION, float3 Norm : NORMAL, float3 Tex0 : TEXCOORD0)
{
    VS_OUTPUT Out = (VS_OUTPUT)0;

    float3 L = -lightDir;
    float3 P = mul(float4(Pos, 1), (float4x3)WorldView);    // position (view space)

    float3 N = normalize(mul(Norm, (float3x3)WorldView));   // normal (view space)

    float3 R = normalize(2 * dot(N, L) * N - L);            // reflection vector (view space)

    float3 V = -normalize(P);                               // view direction (view space)


    Out.Pos  = mul(float4(P, 1), Projection);             // position (projected)

    Out.Diff = I_a * k_a + I_d * k_d * max(0, dot(N, L)); // diffuse + ambient

    Out.Spec = I_s * k_s * pow(max(0, dot(R, V)), n/4);   // specular

    Out.Tex0 = Pos.xz;        

    return Out;
}

VS_OUTPUT VSOUTLINE(float3 Pos : POSITION, float3 Norm : NORMAL, float3 Tex0 : TEXCOORD0)
{
    VS_OUTPUT Out = (VS_OUTPUT)0;

    Pos.xyz = Pos.xyz + 0.01f * Norm.xyz;
    float3 P = mul(float4(Pos, 1), (float4x3)WorldView);    // position (view space)

    
    Out.Pos  = mul(float4(P, 1), Projection);             // position (projected)

    
    return Out;   
}

// samplers

sampler Sampler = sampler_state 
{
    texture = <Tex2>;
    AddressU  = WRAP;        
    AddressV  = WRAP;
    AddressW  = WRAP;
    MIPFILTER = LINEAR;
    MINFILTER = LINEAR;
    MAGFILTER = LINEAR;
};

// samplers

sampler Sampler2 = sampler_state 
{
    texture = <Tex>;
    AddressU  = WRAP;        
    AddressV  = WRAP;
    AddressW  = WRAP;
    MIPFILTER = LINEAR;
    MINFILTER = LINEAR;
    MAGFILTER = LINEAR;
};

PS_OUTPUT PS(VS_OUTPUT In) : COLOR
{   
    PS_OUTPUT output;

    float4 Diff;

    if(In.Diff.x < 0.4f)
        Diff = 0.4f;
    else
        if(In.Diff.x < 0.8f)
            Diff = 0.8f;
        else
            Diff = 1.0f;

    output.col.rgb = (float3(0.75f, 0.0f, 0.0f) - 0.1f * tex2D(Sampler, In.Tex0)) * Diff + In.Spec;
    output.col.w   = 1;

    return output;
}  

PS_OUTPUT PSOUTLINE(VS_OUTPUT In) : COLOR
{
    PS_OUTPUT output;
    output.col = 0;

    return output;
}

technique Tech
{
    pass P0
    {
        CULLMODE = CW;

        VertexShader = compile vs_1_1 VSOUTLINE();
        PixelShader = compile ps_2_0 PSOUTLINE();
    }
    pass P1
    {
        CULLMODE = CCW;

        VertexShader = compile vs_1_1 VS();
        PixelShader  = compile ps_2_0 PS();
    }
}

I'd never really done and shader work before, I just played around with the samples. It's not exceptionally fast, because the actual shading is done by clamping the light values using 'if' statements, instead of the traditional 1D texture.

Anyway, the basic method is:

1) Draw the outline of the object by drawing all of the back facing triangles in black, pushed out slightly along their normals
2) Draw the object itself, switching the cull mode back, but clamp the lighting values to relatively large steps, so that it looks like the image has been drawn with only a couple of shades of ink (like in a cartoon). This can either be done in the shader itself, like I did, which is probably really slow, or with a 1D texture of shades.

I just realised my cel-shading only takes into account the red colour of the object

---------------------------------------

Let's struggle for our dream of Game!

http://andrewporritt.4t.com

[edited by - f8k8 on June 2, 2004 6:21:58 PM]

//

// Cel-Shader

// Note: This effect file works with EffectEdit.

//


string XFile = "teapot.x";
int    BCLR  = 0xFF0080FF;

// transformations

float4x3 WorldView  : WORLDVIEW;
float4x4 Projection : PROJECTION;

// light direction (view space)

float3 lightDir <  string UIDirectional = "Light Direction"; > = {0.577, -0.577, 0.577};

// light intensity

float4 I_a = { 0.3f, 0.3f, 0.3f, 1.0f };    // ambient

float4 I_d = { 1.0f, 1.0f, 1.0f, 1.0f };    // diffuse

float4 I_s = { 0.6f, 0.6f, 0.6f, 1.0f };    // specular


// material reflectivity

float4 k_a = { 0.2f, 0.2f, 0.2f, 1.0f };    // ambient

float4 k_d = { 1.0f, 0.7f, 0.2f, 1.0f };    // diffuse

float4 k_s = { 1.0f, 1.0f, 1.0f, 1.0f };    // specular

float  n   = 64.0f;                         // power


// textures

texture Shades < string function = "shades"; int width = 16; int height = 1; >;
texture Tex < string name = "water.bmp"; >;

float4 shades(float2 Pos : POSITION) : COLOR
{
    float4 output;

    if(Pos.x < 0.4f)
        output.xyzw = 0.4f;
    else
        if(Pos.x < 0.8f)
            output.xyzw = 0.75f;
           else
                output.xyzw = 1.0f;

    return output;
}

struct VS_OUTPUT
{
    float4 Pos  : POSITION;
    float3 Diff : COLOR0;
    float3 Spec : COLOR1;
    float2 Tex0 : TEXCOORD0; 
    float2 Tex1 : TEXCOORD1;             
};

struct PS_OUTPUT
{
    float4 col;
};

VS_OUTPUT VS(float3 Pos : POSITION, float3 Norm : NORMAL, float3 Tex0 : TEXCOORD0)
{
    VS_OUTPUT Out = (VS_OUTPUT)0;

    float3 L = -lightDir;
    float3 P = mul(float4(Pos, 1), (float4x3)WorldView);    // position (view space)

    float3 N = normalize(mul(Norm, (float3x3)WorldView));   // normal (view space)

    float3 R = normalize(2 * dot(N, L) * N - L);            // reflection vector (view space)

    float3 V = -normalize(P);                               // view direction (view space)


    Out.Pos  = mul(float4(P, 1), Projection);             // position (projected)

    Out.Diff = I_a * k_a + I_d * k_d * max(0, dot(N, L)); // diffuse + ambient

    Out.Spec = I_s * k_s * pow(max(0, dot(R, V)), n/4);   // specular

    Out.Tex0 = Pos.xz;        
    Out.Tex1.y = 0.0f;
    Out.Tex1.x = Out.Diff.x + Out.Diff.y + Out.Diff.z;

    return Out;
}

VS_OUTPUT VSOUTLINE(float3 Pos : POSITION, float3 Norm : NORMAL, float3 Tex0 : TEXCOORD0)
{
    VS_OUTPUT Out = (VS_OUTPUT)0;

    Pos.xyz = Pos.xyz + 0.01f * Norm.xyz;
    float3 P = mul(float4(Pos, 1), (float4x3)WorldView);    // position (view space)

    
    Out.Pos  = mul(float4(P, 1), Projection);             // position (projected)

    
    return Out;   
}

sampler ShaderSampler = sampler_state
{
    texture = <Shades>;
    AddressU = CLAMP;
    AddressV  = CLAMP;
    AddressW  = CLAMP;
    MIPFILTER = POINT;
    MINFILTER = POINT;
    MAGFILTER = POINT;
};

sampler TextureSampler = sampler_state
{
    texture = <Tex>;
    AddressU = WRAP;
    AddressV  = WRAP;
    AddressW  = WRAP;
    MIPFILTER = LINEAR;
    MINFILTER = LINEAR;
    MAGFILTER = LINEAR;    
};

PS_OUTPUT PS(VS_OUTPUT In) : COLOR
{   
    PS_OUTPUT output;

    output.col.rgb = tex2D(TextureSampler, In.Tex0) * tex2D(ShaderSampler, In.Tex1);
    output.col.w = 1.0f;

    return output;
}  

PS_OUTPUT PSOUTLINE(VS_OUTPUT In) : COLOR
{
    PS_OUTPUT output;
    output.col = 0;

    return output;
}

technique Tech
{
    pass P0
    {
        CULLMODE = CW;

        VertexShader = compile vs_1_1 VSOUTLINE();
        PixelShader = compile ps_1_1 PSOUTLINE();
    }
    pass P1
    {
        CULLMODE = CCW;

        VertexShader = compile vs_1_1 VS();
        PixelShader  = compile ps_1_1 PS();
    }
}

This is an improved one which generates the 1D texture.

---------------------------------------

Let''s struggle for our dream of Game!

http://andrewporritt.4t.com