Original Post
Don't worry. You won't have to debug my code. But in case you're curious...
// compute a homography matrix
matrix math::matrix_homography( float src_x0, float src_y0, float src_x1, float src_y1, float src_x2, float src_y2, float src_x3, float src_y3,
float dst_x0, float dst_y0, float dst_x1, float dst_y1, float dst_x2, float dst_y2, float dst_x3, float dst_y3)
{
float dx1, dy1, dx2, dy2, sx, sy, g, h, a, b, c, d, e, f;
// quad to square
dx1 = src_x1-src_x2;
dy1 = src_y1-src_y2;
dx2 = src_x3-src_x2;
dy2 = src_y3-src_y2;
sx = src_x0-src_x1+src_x2-src_x3;
sy = src_y0-src_y1+src_y2-src_y3;
g = (sx*dy2-dx2*sy)/(dx1*dy2-dx2*dy1);
h = (dx1*sy-sx*dy1)/(dx1*dy2-dx2*dy1);
a = src_x1-src_x0+g*src_x1;
b = src_x3-src_x0+h*src_x3;
c = src_x0;
d = src_y1-src_y0+g*src_y1;
e = src_y3-src_y0+h*src_y3;
f = src_y0;
matrix quad_to_square_inv;
quad_to_square_inv.m[0][0] = a; quad_to_square_inv.m[0][1] = d; quad_to_square_inv.m[0][2] = 0.0f; quad_to_square_inv.m[0][3] = g;
quad_to_square_inv.m[1][0] = b; quad_to_square_inv.m[1][1] = e; quad_to_square_inv.m[1][2] = 0.0f; quad_to_square_inv.m[1][3] = h;
quad_to_square_inv.m[2][0] = 0.0f; quad_to_square_inv.m[2][1] = 0.0f; quad_to_square_inv.m[2][2] = 1.0f; quad_to_square_inv.m[2][3] = 0.0f;
quad_to_square_inv.m[3][0] = c; quad_to_square_inv.m[3][1] = f; quad_to_square_inv.m[3][2] = 0.0f; quad_to_square_inv.m[3][3] = 1.0f;
matrix quad_to_square = matrix_invert(&quad_to_square_inv);
// square to quad
dx1 = dst_x1-dst_x2;
dy1 = dst_y1-dst_y2;
dx2 = dst_x3-dst_x2;
dy2 = dst_y3-dst_y2;
sx = dst_x0-dst_x1+dst_x2-dst_x3;
sy = dst_y0-dst_y1+dst_y2-dst_y3;
g = (sx*dy2-dx2*sy)/(dx1*dy2-dx2*dy1);
h = (dx1*sy-sx*dy1)/(dx1*dy2-dx2*dy1);
a = dst_x1-dst_x0+g*dst_x1;
b = dst_x3-dst_x0+h*dst_x3;
c = dst_x0;
d = dst_y1-dst_y0+g*dst_y1;
e = dst_y3-dst_y0+h*dst_y3;
f = dst_y0;
matrix square_to_quad;
square_to_quad.m[0][0] = a; square_to_quad.m[0][1] = d; square_to_quad.m[0][2] = 0.0f; square_to_quad.m[0][3] = g;
square_to_quad.m[1][0] = b; square_to_quad.m[1][1] = e; square_to_quad.m[1][2] = 0.0f; square_to_quad.m[1][3] = h;
square_to_quad.m[2][0] = 0.0f; square_to_quad.m[2][1] = 0.0f; square_to_quad.m[2][2] = 1.0f; square_to_quad.m[2][3] = 0.0f;
square_to_quad.m[3][0] = c; square_to_quad.m[3][1] = f; square_to_quad.m[3][2] = 0.0f; square_to_quad.m[3][3] = 1.0f;
return quad_to_square*square_to_quad;
}
...and I'm applying this matrix to the vertices (x, y, z, w) in screen space. The problem is that this matrix modifies the w component of the vertex, which normally isn't a problem because we only really care about x and y (and z, because it is used for perspective projections). BUT OpenGL does care. It uses the w component to compute depth and, therefore, clipping. When I apply a homography matrix to the screen-space vertices, it causes triangles to be unnecessarily clipped (depending on my view angle etc.). Is there any way to take this matrix, and make it leave the w coordinate alone?