Original Post
I've run in to a problem with my "run to selection" algorithm/method. I have a terrain that is hilly, and when I try to do picking so I can place a ball (for testing) at the terrain's location where the cursor ray and plane intersect the ray actually hits multiple planes that are all within the polygon. I was thinking of making some sort of list or vector with all the planes it came in contact with and selecting the closest one, but I want some ideas on how to approach this. I do my picking exactly like what is shown here [http://www.cs.princeton.edu/courses/archive/fall00/cs426/lectures/raycast/sld017.htm]. Only thing I do when I click a spot is capture the mouse message, process it with gluUnProject(), and then run that through the equations shown in the website. The other problem I have is when I do click a spot, half my terrain/map dissapears until I click a different area. Here is what I have that may help. EDIT: I should mention v3one, v3two, and v3three are the three points to the triangle that is being rendered. Each one is a 3 dimensional array.
//MOUSE MESSAGE
case WM_LBUTTONDOWN:
{
LPPOINT pt = new POINT;
GetCursorPos(pt);
ScreenToClient(hwnd,pt);
if(!CalcMouseRay(pt, Ray, Po)) MessageBox(hwnd,"Error calculating mouse cursor ray!","test",MB_OK);
}
break;
//FUNCTIONS USED
bool CalcMouseRay(LPPOINT cCoords, vec3 Ray, vec3 Po)
{
bool bSuccess = true;
glvec3 p1, p2;
double mvMatrix[16];
double pMatrix[16];
int viewport[4];
glGetDoublev(GL_MODELVIEW_MATRIX,mvMatrix);
glGetDoublev(GL_PROJECTION_MATRIX,pMatrix);
glGetIntegerv(GL_VIEWPORT,viewport);
if(!gluUnProject(cCoords->x,cCoords->y,0,mvMatrix,pMatrix,viewport,&p1[0],&p1[1],&p1[2])) bSuccess = false;
if(!gluUnProject(cCoords->x,cCoords->y,1,mvMatrix,pMatrix,viewport,&p2[0],&p2[1],&p2[2])) bSuccess = false;
Po[0] = p1[0];
Po[1] = p1[1];
Po[2] = p1[2];
Ray[0] = p2[0] - p1[0];
Ray[1] = p2[1] - p1[1];
Ray[2] = p2[2] - p1[2];
return bSuccess;
}
float CalcD(vec3 Po, vec3 N)
{
float z = Po[2] + ( N[0]*Po[0] + N[1]*Po[1] ) / N[2];
return N[2]*z;
}
bool DrawIntersectPoint(vec3 P)
{
glPointSize(50);
glBegin(GL_POINT);
glVertex3fv(P);
glEnd();
glPointSize(2);
return true;
}
//IN THE RENDER FUNCTION TO CHECK FOR THE PLANES
if(type == MDL_COLLIDE_TERRAIN){
vec3 normal, tdRay;
float NPod, RayN, d;
GetNormal(v3one,v3two,v3three,normal);
d = CalcD(v3one,normal);
NPod = v3DotProduct(normal,v3one) + d;
RayN = v3DotProduct(Ray,normal);
tdRay[0] = Ray[0]*NPod/RayN;
tdRay[1] = Ray[1]*NPod/RayN;
tdRay[2] = Ray[2]*NPod/RayN;
Point[0] = Po[0]-tdRay[0];
Point[1] = Po[1]-tdRay[1];
Point[2] = Po[2]-tdRay[2];
if(WithinPolygon(Point, v3one, v3two, v3three)){
DrawIntersectPoint(Point);
}
}