Original Post
Hello, first of all I want to apologize for I'm not a native speaker, but will do my best so you can understand me, thank you! I have following very simple scene: 
What I'm trying to do is to detect the red edge (in this case v0-v2) in geometry shader and then eventually extrude it along the light rays to achieve a simple shadow polygon (but that's not my problem now). The input into the geometry shader is the whole triangle v0-v1-v2 and all I need to do is to detect somehow the red edge. I have tried to detect it by comparing angles between vectors from the light to the edge vertices in camera's clip space; on the picture the angle between light->v0 and light->v2 is bigger than between the other two combinations, so there should be the red edge. It sometimes seems to work for some camera positions but generally it fails and I really don't know where the mistake could be. Here is my vertex shader: Here is my geometry shader: Maybe my whole approach is wrong, so please tell me how can I detect the edge? If my approach is not completely wrong, where do I have the mistake? Thank you very much! Equilibrix
What I'm trying to do is to detect the red edge (in this case v0-v2) in geometry shader and then eventually extrude it along the light rays to achieve a simple shadow polygon (but that's not my problem now). The input into the geometry shader is the whole triangle v0-v1-v2 and all I need to do is to detect somehow the red edge. I have tried to detect it by comparing angles between vectors from the light to the edge vertices in camera's clip space; on the picture the angle between light->v0 and light->v2 is bigger than between the other two combinations, so there should be the red edge. It sometimes seems to work for some camera positions but generally it fails and I really don't know where the mistake could be. Here is my vertex shader:
varying vec3 lightToVertexDirection;
void main()
{
// get vertex in clip space
vec4 vertexClip = ftransform();
// get light in clip space
vec4 lightClip = gl_ProjectionMatrix * gl_LightSource[0].position;
// get direction
lightToVertexDirection = normalize(vertexClip.xyz - lightClip.xyz);
gl_Position = gl_ModelViewMatrix * gl_Vertex;
}
#version 120
#extension GL_EXT_geometry_shader4 : enable
#extension GL_EXT_gpu_shader4 : enable
varying in vec3 lightToVertexDirection[];
void main()
{
// get light position in camera's eye space
vec4 lightPos = gl_LightSource[0].position;
// compute normal at each vertex
vec3 n[3];
n[0] = cross(normalize(gl_PositionIn[1].xyz - gl_PositionIn[0].xyz), normalize(gl_PositionIn[2].xyz - gl_PositionIn[0].xyz));
n[1] = cross(normalize(gl_PositionIn[2].xyz - gl_PositionIn[1].xyz), normalize(gl_PositionIn[0].xyz - gl_PositionIn[1].xyz));
n[2] = cross(normalize(gl_PositionIn[0].xyz - gl_PositionIn[2].xyz), normalize(gl_PositionIn[1].xyz - gl_PositionIn[2].xyz));
// compute direction from vertex to light position
vec3 dir[3];
dir[0] = normalize(gl_PositionIn[0].xyz - lightPos.xyz);
dir[1] = normalize(gl_PositionIn[1].xyz - lightPos.xyz);
dir[2] = normalize(gl_PositionIn[2].xyz - lightPos.xyz);
// continue only if this triangle is not facing the light
if (!(dot(n[0], dir[0]) > 0 || dot(n[1], dir[1]) > 0 || dot(n[2], dir[2]) > 0))
{
return;
}
gl_FrontColor = vec4(1.0, 1.0, 0.0, 0.0);
// draw lines from light to vertices (just for visualisation purposes)
for (int i = 0; i < gl_VerticesIn; ++i)
{
gl_FrontColor = vec4(1.0, 1.0, 0.0, 0.0);
gl_Position = gl_ProjectionMatrix * gl_PositionIn;
EmitVertex();
gl_Position = gl_ProjectionMatrix * lightPos;
EmitVertex();
EndPrimitive();
}
// normalize the light->vertex direction in clip space
vec3 dirx[3];
dirx[0] = normalize(lightToVertexDirection[0]);
dirx[1] = normalize(lightToVertexDirection[1]);
dirx[2] = normalize(lightToVertexDirection[2]);
// detect the edge of the shadow polygon
for (int i = 0; i < gl_VerticesIn; ++i)
{
// compute indices for the other two vertices
int i0 = i;
int i1 = (i+1) % 3;
int i2 = (i+2) % 3;
// get the angles between light and both edge vertices
float d0 = acos(dot(dirx[i0], dirx[i1]));
float d1 = acos(dot(dirx[i0], dirx[i2]));
float d2 = acos(dot(dirx[i1], dirx[i2]));
// if currently processed edge has the largest angle, it is the edge!
if (d0 >= d1 && d0 >= d2)
{
gl_FrontColor = vec4(0.0, 1.0, 0.0, 0.0);
// draw the edge
gl_Position = gl_ProjectionMatrix * gl_PositionIn[i0];
EmitVertex();
gl_Position = gl_ProjectionMatrix * gl_PositionIn[i1];
EmitVertex();
EndPrimitive();
return;
}
}
}
