Skip to main content
GameDev.net gamedev.net
🔒 Locked

Effect fails to compile unless preshader disabled

Started by jrh2365 Jul 3, 2013 at 12:30 PM 2 replies 1.4k views
Original Post
jrh2365
jrh2365

I have an effect that I'm using in XNA that fails to compile in release mode, but works fine in debug mode. I was able to reproduce the behavior using the version of fxc included in the March 2009 DirectX SDK. Compilation fails unless preshaders are disabled (/Op)

This is the output, which isn't particularly helpful


error : (202,17): ID3DXEffectCompiler::CompileEffect: There was an error compiling expression
error : ID3DXEffectCompiler: Compilation failed 

The problem seems related to the if/elseif/else block at the top of TransformPixel().

Any suggestions? XNA doesn't seem to have a way to disable preshaders but keep other optimizations. It also applies some post processing to the compiled effect code. I was thinking I could replace XNA's EffectProcessor with one that uses a compiler from a more recent DirectX SDK, and then call XNA's post processor through reflection. I did a quick test of this and was able to load an effect compiled using the June 2010 SDK, but I'm not sure if there are cases where the output isn't compatible.

Here is the effect code, from http://animationcomponents.codeplex.com/


float4x4 World;
float4x4 View;
float4x4 Projection;
float3 DiffuseColor;
float3 SpecularColor;
float3 AmbientLightColor = float3(0,0,0);
float3 EmissiveColor;
float3 EyePosition;
float3 FogColor;
bool   FogEnable;
float FogStart;
float FogEnd;
bool   DirLight0Enable;
bool   DirLight1Enable;
extern bool    DirLight2Enable;
float3 DirLight0Direction;
float3 DirLight1Direction;
float3 DirLight2Direction;
float3 DirLight0DiffuseColor;
float3 DirLight1DiffuseColor;
float3 DirLight2DiffuseColor;
float3 DirLight0SpecularColor;
float3 DirLight1SpecularColor;
float3 DirLight2SpecularColor;
float Alpha;
float4x4 MatrixPalette[56];
float SpecularPower;
bool TextureEnabled;
bool LightingEnable = false;
texture BasicTexture;

sampler TextureSampler = sampler_state
{
   Texture = (BasicTexture);

};

struct VS_INPUT
{
    float4 position : POSITION;
    float4 color : COLOR;
    float2 texcoord : TEXCOORD0;
    float3 normal : NORMAL0;
    half4 indices : BLENDINDICES0;
    float4 weights : BLENDWEIGHT0;
};

struct VS_OUTPUT
{
    float4 position : POSITION;
    float4 color : COLOR;
    float2 texcoord : TEXCOORD0;
    float  distance : TEXCOORD1;
};

struct PS_INPUT
{
    float4 color : COLOR;
    float2 texcoord : TEXCOORD0;
    float  distance : TEXCOORD1;
};

struct PS_OUTPUT
{
    float4 color : COLOR;
};

struct SKIN_OUTPUT
{
    float4 position;
    float4 normal;
};

SKIN_OUTPUT Skin4( const VS_INPUT input)
{
    SKIN_OUTPUT output = (SKIN_OUTPUT)0;

    float lastWeight = 1.0;
    float weight = 0;
    for (int i = 0; i < 3; ++i)
    {
        weight = input.weights[i];
        lastWeight -= weight;
        output.position     += mul( input.position, MatrixPalette[input.indices[i]]) * weight;
        output.normal       += mul( input.normal  , MatrixPalette[input.indices[i]]) * weight;
    }
    output.position     += mul( input.position, MatrixPalette[input.indices[3]])*lastWeight;
    output.normal       += mul( input.normal  , MatrixPalette[input.indices[3]])*lastWeight;
    return output;
};

void TransformVertex (in VS_INPUT input, out VS_OUTPUT output)
{

    float3 inputN = normalize(input.normal);
    SKIN_OUTPUT skin = Skin4(input);
    output.position=skin.position;
    float3 normal = skin.normal;

    normal = normalize(mul(normal,World));

    float3 totalDiffuse = DiffuseColor*
         ((DirLight0Enable ? dot(-DirLight0Direction,normal) * DirLight0DiffuseColor : 0) +
         (DirLight1Enable ? dot(-DirLight1Direction,normal) * DirLight1DiffuseColor : 0) +
         (DirLight2Enable ?  dot(-DirLight2Direction,normal) * DirLight2DiffuseColor : 0));

    float3 viewDirection = normalize(EyePosition - mul(output.position,World));

    float3 spec0,spec1,spec2;
    if (DirLight0Enable)
    {
        float val = dot(-DirLight0Direction,normal);
        if (val < 0)
        {
            spec0 = float3(0,0,0);
        }
        else
        {
            spec0 = DirLight0SpecularColor *
                (pow(val*dot(reflect(DirLight0Direction,normal),viewDirection),SpecularPower));
        }
    }
    else
        spec0=float3(0,0,0);

    if (DirLight1Enable)
    {
        float val = dot(-DirLight1Direction,normal);
        if (val < 0)
        {
            spec1 = float3(0,0,0);
        }
        else
        {
            spec1 = DirLight1SpecularColor *
                (pow(val*dot(reflect(DirLight1Direction,normal),viewDirection),SpecularPower));
        }
    }
    else
        spec1=float3(0,0,0);

    if (DirLight2Enable)
    {
        float val = dot(-DirLight2Direction,normal);
        if (val < 0)
        {
            spec2 = float3(0,0,0);
        }
        else
        {
            spec2 = DirLight2SpecularColor *
                (pow(val*dot(reflect(DirLight2Direction,normal),viewDirection),SpecularPower));
        }
    }
    else
        spec2=float3(0,0,0);
    
    float3 totalSpecular = SpecularColor * (spec0+spec1+spec2);
    output.color.xyz = saturate(AmbientLightColor+totalDiffuse + totalSpecular);
    output.color.w=1.0;
    output.texcoord = input.texcoord;
    output.position = mul(output.position,World);
    output.distance = distance(EyePosition, output.position.xyz);
    output.position = mul(output.position,mul(View,Projection));
}



void TransformPixel (in PS_INPUT input, out PS_OUTPUT output)
{
    if (LightingEnable == false && TextureEnabled)
    {
        output.color.xyz = tex2D(TextureSampler,input.texcoord).xyz * saturate(EmissiveColor + DiffuseColor);
    }
    else if (LightingEnable == false)
    {
       output.color.xyz = saturate(EmissiveColor + DiffuseColor);
    }
    else
    {
        output.color.xyz = TextureEnabled ? tex2D(TextureSampler, input.texcoord).xyz  * input.color.xyz
            : input.color.xyz;
    }
    output.color.w   = 
         TextureEnabled ? tex2D(TextureSampler, input.texcoord).w * Alpha : Alpha;
    
    if (FogEnable)
    {
        float dist = (input.distance - FogStart) / (FogEnd - FogStart);
        dist = saturate(dist);
        float3 distv = float3(dist,dist,dist);
        distv = lerp(output.color.xyz,FogColor,distv);
        output.color.xyz = distv;
    }
}

technique TransformTechnique
{
    pass P0
    {
        VertexShader = compile vs_2_0 TransformVertex();
        PixelShader  = compile ps_2_0 TransformPixel();
    }
}
phil_t
phil_t

Looks like it's getting confused by having to put saturate in the pre-shader (maybe that's not supported in preshaders?).

If I were you, I'd work around it by adding another shader variable that just has the value of saturate(EmissiveColor + DiffuseColor). That'd be a lot easier that writing your own effect processor.

AmzBee
AmzBee

I can get this shader to compile in release when I make a slight alteration. It seems the lines 173 and 177 are causing issue, try removing the following from each:


saturate(EmissiveColor + DiffuseColor)

At first I thought saturate was causing the issue but you are using it on line 190 as well so it's ruled out. After a few mins of tweaks, I found the following works as an unusual solution:


float4x4 World;
float4x4 View;
float4x4 Projection;
float3 DiffuseColor;
float3 SpecularColor;
float3 AmbientLightColor = float3(0,0,0);
float3 EmissiveColor;
float3 EyePosition;
float3 FogColor;
bool   FogEnable;
float FogStart;
float FogEnd;
bool   DirLight0Enable;
bool   DirLight1Enable;
extern bool    DirLight2Enable;
float3 DirLight0Direction;
float3 DirLight1Direction;
float3 DirLight2Direction;
float3 DirLight0DiffuseColor;
float3 DirLight1DiffuseColor;
float3 DirLight2DiffuseColor;
float3 DirLight0SpecularColor;
float3 DirLight1SpecularColor;
float3 DirLight2SpecularColor;
float Alpha;
float4x4 MatrixPalette[56];
float SpecularPower;
bool TextureEnabled;
bool LightingEnable = false;
texture BasicTexture;

sampler TextureSampler = sampler_state
{
   Texture = (BasicTexture);

};

struct VS_INPUT
{
    float4 position : POSITION;
    float4 color : COLOR;
    float2 texcoord : TEXCOORD0;
    float3 normal : NORMAL0;
    half4 indices : BLENDINDICES0;
    float4 weights : BLENDWEIGHT0;
};

struct VS_OUTPUT
{
    float4 position : POSITION;
    float4 color : COLOR0;
    float2 texcoord : TEXCOORD0;
    float  distance : TEXCOORD1;
	float3 diffAndEmissive : COLOR1;
};

struct PS_INPUT
{
    float4 color : COLOR0;
    float2 texcoord : TEXCOORD0;
    float  distance : TEXCOORD1;
	float3 diffAndEmissive : COLOR1;
};

struct PS_OUTPUT
{
    float4 color : COLOR;
};

struct SKIN_OUTPUT
{
    float4 position;
    float4 normal;
};

SKIN_OUTPUT Skin4( const VS_INPUT input)
{
    SKIN_OUTPUT output = (SKIN_OUTPUT)0;

    float lastWeight = 1.0;
    float weight = 0;
    for (int i = 0; i < 3; ++i)
    {
        weight = input.weights[i];
        lastWeight -= weight;
        output.position     += mul( input.position, MatrixPalette[input.indices[i]]) * weight;
        output.normal       += mul( input.normal  , MatrixPalette[input.indices[i]]) * weight;
    }
    output.position     += mul( input.position, MatrixPalette[input.indices[3]])*lastWeight;
    output.normal       += mul( input.normal  , MatrixPalette[input.indices[3]])*lastWeight;
    return output;
};

void TransformVertex (in VS_INPUT input, out VS_OUTPUT output)
{

    float3 inputN = normalize(input.normal);
    SKIN_OUTPUT skin = Skin4(input);
    output.position=skin.position;
    float3 normal = skin.normal;

    normal = normalize(mul(normal,World));

    float3 totalDiffuse = DiffuseColor*
         ((DirLight0Enable ? dot(-DirLight0Direction,normal) * DirLight0DiffuseColor : 0) +
         (DirLight1Enable ? dot(-DirLight1Direction,normal) * DirLight1DiffuseColor : 0) +
         (DirLight2Enable ?  dot(-DirLight2Direction,normal) * DirLight2DiffuseColor : 0));

    float3 viewDirection = normalize(EyePosition - mul(output.position,World));

    float3 spec0,spec1,spec2;
    if (DirLight0Enable)
    {
        float val = dot(-DirLight0Direction,normal);
        if (val < 0)
        {
            spec0 = float3(0,0,0);
        }
        else
        {
            spec0 = DirLight0SpecularColor *
                (pow(val*dot(reflect(DirLight0Direction,normal),viewDirection),SpecularPower));
        }
    }
    else
        spec0=float3(0,0,0);

    if (DirLight1Enable)
    {
        float val = dot(-DirLight1Direction,normal);
        if (val < 0)
        {
            spec1 = float3(0,0,0);
        }
        else
        {
            spec1 = DirLight1SpecularColor *
                (pow(val*dot(reflect(DirLight1Direction,normal),viewDirection),SpecularPower));
        }
    }
    else
        spec1=float3(0,0,0);

    if (DirLight2Enable)
    {
        float val = dot(-DirLight2Direction,normal);
        if (val < 0)
        {
            spec2 = float3(0,0,0);
        }
        else
        {
            spec2 = DirLight2SpecularColor *
                (pow(val*dot(reflect(DirLight2Direction,normal),viewDirection),SpecularPower));
        }
    }
    else
        spec2=float3(0,0,0);
    
    float3 totalSpecular = SpecularColor * (spec0+spec1+spec2);
    output.color.xyz = saturate(AmbientLightColor+totalDiffuse + totalSpecular);
    output.color.w=1.0;
    output.texcoord = input.texcoord;
    output.position = mul(output.position,World);
    output.distance = distance(EyePosition, output.position.xyz);
    output.position = mul(output.position,mul(View,Projection));
	output.diffAndEmissive = EmissiveColor + DiffuseColor;
}



void TransformPixel (in PS_INPUT input, out PS_OUTPUT output)
{
    if (LightingEnable == false && TextureEnabled)
    {
        output.color.xyz = tex2D(TextureSampler,input.texcoord).xyz * saturate(input.diffAndEmissive);
    }
    else if (LightingEnable == false)
    {
       output.color.xyz = saturate(input.diffAndEmissive);
    }
    else
    {
        output.color.xyz = TextureEnabled ? tex2D(TextureSampler, input.texcoord).xyz  * input.color.xyz
            : input.color.xyz;
    }
    output.color.w   = 
         TextureEnabled ? tex2D(TextureSampler, input.texcoord).w * Alpha : Alpha;
    
    if (FogEnable)
    {
        float dist = (input.distance - FogStart) / (FogEnd - FogStart);
        dist = saturate(dist);
        float3 distv = float3(dist,dist,dist);
        distv = lerp(output.color.xyz,FogColor,distv);
        output.color.xyz = distv;
    }
}

technique TransformTechnique
{
    pass P0
    {
        VertexShader = compile vs_2_0 TransformVertex();
        PixelShader  = compile ps_2_0 TransformPixel();
    }
}

Note that the only difference is adding the emissive and diffuse colour is done on the vertex shader instead and passed along to the pixel shader. I am aware this may not be the most ideal solution, but hopefully it'll help shed some light on how to get around the issue :)

Aimee

We are now on Tumblr, so please come and check out our site!    http://xpod-games.com
jrh2365
jrh2365

Thanks, pulling the saturate calls out and using another effect parameter works fine.

Topic Locked

This topic has been locked by a moderator. New replies are not allowed.

Sign in to reply to this topic.