Normal-Mapping Woes

This is my shader:



float mix( in float _fX, in float _fY, in float _fA ) { return _fX + _fA * (_fY - _fX); }
float2 mix( in float2 _fX, in float2 _fY, in float _fA ) { return _fX + _fA * (_fY - _fX); }
float3 mix( in float3 _fX, in float3 _fY, in float _fA ) { return _fX + _fA * (_fY - _fX); }
float4 mix( in float4 _fX, in float4 _fY, in float _fA ) { return _fX + _fA * (_fY - _fX); }
float2 mix( in float2 _fX, in float2 _fY, in float2 _fA ) { return _fX + _fA * (_fY - _fX); }
float3 mix( in float3 _fX, in float3 _fY, in float3 _fA ) { return _fX + _fA * (_fY - _fX); }
float4 mix( in float4 _fX, in float4 _fY, in float4 _fA ) { return _fX + _fA * (_fY - _fX); }

matrix<float, 4, 4> g_mViewMatrix;
matrix<float, 3, 3> g_mNormalMatrix;
vector<float, 4> g_vDiffuseMaterial;
vector<float, 4> g_vAmbientMaterial;
vector<float, 4> g_vEmissiveMaterial;
vector<float, 4> g_vSpecularMaterial;
float g_fPower;
int g_iTotalDirLights;
int g_iTotalPointLights;
vector<float, 4> g_vLightVectors[8];
vector<float, 4> g_vLightHalfVectors[8];
vector<float, 4> g_vLightAmbients[8];
vector<float, 4> g_vLightDiffuses[8];
vector<float, 4> g_vLightSpeculars[8];
vector<float, 4> g_vLightAttens[8];
vector<float, 4> g_vLightRanges[8];
vector<float, 3> g_vSpotLightDirs[8];
vector<float, 4> g_vSkyColor;
vector<float, 4> g_vGroundColor;
sampler2D g_sSampler2dTex0;
sampler2D g_sSampler2dTex1;
sampler2D g_sSampler2dTex2;
sampler2D g_sSampler2dTex3;
sampler2D g_sSampler2dTex4;
sampler2D g_sSampler2dTex5;
sampler2D g_sSampler2dTex6;
sampler2D g_sSampler2dTex7;

struct LSE_COLOR_PAIR {
vector<float, 4> cDiffuse;
vector<float, 4> cSpecular;
};


LSE_COLOR_PAIR GetDirLightColors(vector<float, 3> _vNormalInViewSpace, vector<float, 4> _vPosVector, int _iIndex){
// Removed to save space.
}

void Main(in vector<float, 2> _vIn2dTex0:TEXCOORD2, in vector<float, 3> _vInNormal:NORMAL0, in vector<float, 3> _vInTangent:TANGENT1, in vector<float, 3> _vInBiNormal:BINORMAL1, in vector<float, 4> _vInPos:POSITION0, in vector<float, 4> _vInEyePos:TEXCOORD1, out vector<float, 4> _vOutColor:COLOR0){
// Determine the normal. Read from the normal map and offset the input normal by that amount.
vector<float, 3> vNormalizedNormal = tex2D(g_sSampler2dTex1, _vIn2dTex0).xyz;
vNormalizedNormal = ((vNormalizedNormal*2.0)-1.0);
vNormalizedNormal = normalize(((normalize(_vInNormal) + (vNormalizedNormal.x*_vInTangent)) + (vNormalizedNormal.y*_vInBiNormal)));

// For calculating the specular component of lighting.
vector<float, 4> vViewPosToEye = -normalize(_vInEyePos);

LSE_COLOR_PAIR cpLightColors = {
vector<float, 4>(0.0, 0.0, 0.0, 0.0),
vector<float, 4>(0.0, 0.0, 0.0, 0.0)
};
for(int I = 0; I < g_iTotalDirLights; I++) {
LSE_COLOR_PAIR cpThis = GetDirLightColors(vNormalizedNormal, vViewPosToEye, I);
cpLightColors.cDiffuse += cpThis.cDiffuse;
cpLightColors.cSpecular += cpThis.cSpecular;
}

// Apply hemisphere ambient lighting.
vector<float, 3> vUp = mul(g_mViewMatrix, vector<float, 4>(0.0, 1.0, 0.0, 0.0)).xyz;
cpLightColors.cDiffuse.xyz += mix(g_vGroundColor.xyz, g_vSkyColor.xyz, ((dot(vUp, vNormalizedNormal) * 0.5) + 0.5));

// Sample the diffuse color from the texture.
_vOutColor = vector<float, 4>(tex2D(g_sSampler2dTex0, _vIn2dTex0).xyz, g_vDiffuseMaterial.w);
_vOutColor.xyz *= g_vDiffuseMaterial.xyz;

// Apply lighting.
_vOutColor.xyz = ((_vOutColor.xyz * cpLightColors.cDiffuse.xyz) + (g_vSpecularMaterial*cpLightColors.cSpecular).xyz);
}

In this form it produces this incorrect result:
http://imageshack.us...badnormals.png/

This is how it should look:
http://imageshack.us...oodnormals.png/


The good image is taken from the OpenGL version of my engine, which uses exactly the same shader for rendering (but translated to GLSL, not HLSL). For the nay-sayers, here is the GLSL version of the exact same shader:


#version 130
precision mediump float;
out vec4 _vOutColor;
in vec4 LSG_VERT_OUT_PIXEL_IN_369_1/*_vInEyePos*/;
in vec4 LSG_VERT_OUT_PIXEL_IN_365_0/*_vInPos*/;
in vec3 LSG_VERT_OUT_PIXEL_IN_360_1/*_vInBiNormal*/;
in vec3 LSG_VERT_OUT_PIXEL_IN_368_1/*_vInTangent*/;
in vec3 LSG_VERT_OUT_PIXEL_IN_364_0/*_vInNormal*/;
in vec2 LSG_VERT_OUT_PIXEL_IN_369_2/*_vIn2dTex0*/;

uniform mat4x4 g_mViewMatrix;
uniform mat3x3 g_mNormalMatrix;
uniform vec4 g_vDiffuseMaterial;
uniform vec4 g_vAmbientMaterial;
uniform vec4 g_vEmissiveMaterial;
uniform vec4 g_vSpecularMaterial;
uniform float g_fPower;
uniform int g_iTotalDirLights;
uniform int g_iTotalPointLights;
uniform vec4 g_vLightVectors[8];
uniform vec4 g_vLightHalfVectors[8];
uniform vec4 g_vLightAmbients[8];
uniform vec4 g_vLightDiffuses[8];
uniform vec4 g_vLightSpeculars[8];
uniform vec4 g_vLightAttens[8];
uniform vec4 g_vLightRanges[8];
uniform vec3 g_vSpotLightDirs[8];
uniform vec4 g_vSkyColor;
uniform vec4 g_vGroundColor;
uniform sampler2D g_sSampler2dTex0;
uniform sampler2D g_sSampler2dTex1;
uniform sampler2D g_sSampler2dTex2;
uniform sampler2D g_sSampler2dTex3;
uniform sampler2D g_sSampler2dTex4;
uniform sampler2D g_sSampler2dTex5;
uniform sampler2D g_sSampler2dTex6;
uniform sampler2D g_sSampler2dTex7;

struct LSE_COLOR_PAIR{
vec4 cDiffuse;
vec4 cSpecular;
};


LSE_COLOR_PAIR GetDirLightColors(in vec3 _vNormalInViewSpace, in vec4 _vPosVector, in int _iIndex){
// Removed to save space.
}


void main(){
// Determine the normal. Read from the normal map and offset the input normal by that amount.
vec3 vNormalizedNormal = vec3(texture(g_sSampler2dTex1, LSG_VERT_OUT_PIXEL_IN_369_2/*_vIn2dTex0*/).xyz);
vNormalizedNormal = ((vNormalizedNormal*2.0)-1.0);
vNormalizedNormal = normalize(((normalize(LSG_VERT_OUT_PIXEL_IN_364_0/*_vInNormal*/) + (vNormalizedNormal.x*LSG_VERT_OUT_PIXEL_IN_368_1/*_vInTangent*/)) + (vNormalizedNormal.y*LSG_VERT_OUT_PIXEL_IN_360_1/*_vInBiNormal*/)));

// For calculating the specular component of lighting.
vec4 vViewPosToEye = vec4(-normalize(LSG_VERT_OUT_PIXEL_IN_369_1/*_vInEyePos*/));

LSE_COLOR_PAIR cpLightColors = LSE_COLOR_PAIR( vec4(0.0, 0.0, 0.0, 0.0), vec4(0.0, 0.0, 0.0, 0.0) );

for(int I = 0; I < g_iTotalDirLights; I++){
LSE_COLOR_PAIR cpThis = GetDirLightColors(vNormalizedNormal, vViewPosToEye, I);
cpLightColors.cDiffuse += cpThis.cDiffuse;
cpLightColors.cSpecular += cpThis.cSpecular;
}

// Apply hemisphere ambient lighting.
vec3 vUp = vec3(((g_mViewMatrix) * (vec4(0.0, 1.0, 0.0, 0.0))).xyz);
cpLightColors.cDiffuse.xyz += mix(g_vGroundColor.xyz, g_vSkyColor.xyz, ((dot(vUp, vNormalizedNormal) * 0.5) + 0.5));

// Sample the diffuse color from the texture.
_vOutColor = vec4(texture(g_sSampler2dTex0, LSG_VERT_OUT_PIXEL_IN_369_2/*_vIn2dTex0*/).xyz, g_vDiffuseMaterial.w);
_vOutColor.xyz *= g_vDiffuseMaterial.xyz;

// Apply lighting.
_vOutColor.xyz = ((_vOutColor.xyz * cpLightColors.cDiffuse.xyz) + (g_vSpecularMaterial*cpLightColors.cSpecular).xyz);
}



So indeed the shaders match in logic exactly.
So I checked my inputs.
I exported all of my OpenGL vertex buffers and then added some temporary code to load them into the Direct3D 9 vertex buffers, which ensured that the two sets of vertex buffers were all exactly alike. The result was the same.



So I changed the out color to show me tangents, binormals, input normals, everything.
Input normals matched but the tangent and binormal were wrong.

This is how I transform the tangent (and likewise with the binormal) in the vertex shader:
_vOutTangent = normalize(mul(g_mNormalMatrix, _vInTangent)); HLSL
LSG_VERT_OUT_PIXEL_IN_360_1/*_vOutBiNormal*/ = normalize(((g_mNormalMatrix) * (_vInBiNormal))); GLSL
If I change it to this:
_vOutTangent = _vInTangent;
…and then change the output color in the pixel shader to the tangent value, the result is the same in both HLSL and GLSL. That means initially the input is the same. The data coming in from the vertex buffers is fine. It gets lost when transformed.


The input normals are transformed the same way, and they are always a match between GLSL and HLSL.
Of course, the order of my matrix multiplication is correct, or else I would never be able to get vertices, normals, etc. to match between them.
I have tried transposing the matrices before transforming the tangent and binormal, and also reversing the order of the multiply. The result is the same.



Any ideas?
Yogurt Emperor



[EDIT]
Made the code more readable for those among us who may be human.
[/EDIT]