Gldrawpixels To Stencil Buffer

I am not familiar with OpenCV, but I am more familiar with OpenGL. At a glance, I could not see anything wrong with the gl calls you provided, so I set up a dummy project to prove it. Under OpenGL 4.5, using glew-2.0.0 and glfw-3.2 as a wrapper, I was successful in this proof. My test loads the first image you provided as Stencil.bmp, writes it to a bound buffer, reads it back frm that buffer, and writes the result to disk. I set up a few dummy objects in order to prove I could use your gl code exactly as-is, without modifications. I have included my source (I was not allowed to attach a .cpp file, so I included it just as a code-block below), which produces the expected output as StencilGPU.bmp

These results indicate that your problem lies somewhere in the OpenCV object, not in your OpenGL calls. Perhaps the mat is storing the image in something akin to compression blocks (which are not sequential), and thus you end up writing a bunch of nonsense to the GPU after the end of the first block? You might consider spending some time validating that the buffer you are reading from is both sequential, and also starts from the first scanline (positive stride). You would also need to validate that the buffer you are writing to is both sequential, and also starts from the first scanline (positive stride). It is also possible that just removing OpenCV altogether from the equation in this section of your code would be simpler.

Good luck!

//Include GLEW  
#include <GL/glew.h>  

//Include GLFW  
#include <GLFW/glfw3.h>  

//Include the standard C++ headers  
#include <stdio.h>  
#include <stdlib.h>  
#include <Windows.h>
#include <assert.h>
#include <gdiplus.h>

using namespace Gdiplus;

typedef struct RadDim_Dummy_t
{
	int x, y;
} RadDim_Dummy;

typedef struct Mat_Dummy_t
{
	void* data;
} Mat_Dummy;

int GetEncoderClsid(const WCHAR* format, CLSID* pClsid)
{
	UINT  num = 0;          // number of image encoders
	UINT  size = 0;         // size of the image encoder array in bytes

	ImageCodecInfo* pImageCodecInfo = NULL;

	GetImageEncodersSize(#, &size);
	if(size == 0)
		return -1;  // Failure

	pImageCodecInfo = (ImageCodecInfo*)(malloc(size));
	if(pImageCodecInfo == NULL)
		return -1;  // Failure

	GetImageEncoders(num, size, pImageCodecInfo);

	for(UINT j = 0; j < num; ++j)
	{
		if( wcscmp(pImageCodecInfo[j].MimeType, format) == 0 )
		{
			*pClsid = pImageCodecInfo[j].Clsid;
			free(pImageCodecInfo);
			return j;  // Success
		}
	}
	free(pImageCodecInfo);
	return -1;  // Failure
}

void SetupStencilBuffer()
{
	// load the file copied from gamedev into a gdi+ bitmap
	HBITMAP hBitmap = (HBITMAP)LoadImage( NULL, L"Stencil.bmp", IMAGE_BITMAP, 0, 0, LR_CREATEDIBSECTION | LR_DEFAULTSIZE | LR_LOADFROMFILE );
	BITMAP bmp;
	GetObject(hBitmap, sizeof(BITMAP), &bmp);
	Bitmap *image = new Bitmap(hBitmap, NULL);
	assert(image);
	assert(image->GetPixelFormat() == PixelFormat8bppIndexed);
	assert(GetPixelFormatSize(image->GetPixelFormat()) == 8);
	const int width = image->GetWidth(), height = image->GetHeight(), size = width * height;

	// grab the buffer itself from the gdi+ bitmap
	BitmapData data;
	Rect rect = Rect(0, 0, width, height);
	Status result = image->LockBits(▭, ImageLockModeRead | ImageLockModeWrite, image->GetPixelFormat(), &data);
	assert(result == Ok);

	// flip the buffer if the scanlines are inverted.
	if (data.Stride < 0)
	{
		image->UnlockBits(&data);
		image->RotateFlip(RotateNoneFlipY);
		Status result = image->LockBits(▭, ImageLockModeRead | ImageLockModeWrite, image->GetPixelFormat(), &data);
		assert(result == Ok);
	}
	assert(data.Stride > 0);

	// allocate and initialize a buffer to copy into from glReadPixels
	UINT8* buffer = new UINT8[size];
	memset(buffer, 0, size);

	// setup some dummy variables so i can copy-paste code without any modifications from gamedev
	GLuint _renderStencilBuffer;
	RadDim_Dummy _radDim = { width, height };
	Mat_Dummy stencilImg = { data.Scan0 };
	Mat_Dummy stencilImgRead = { buffer };

	// start sample code from gamedev
	{
		glPixelStorei(GL_UNPACK_ALIGNMENT,1);
		glPixelStorei(GL_UNPACK_ROW_LENGTH,_radDim.x);
		glPixelStorei(GL_UNPACK_SKIP_ROWS,0);
		glPixelStorei(GL_UNPACK_SKIP_PIXELS,0);

		glPixelStorei(GL_PACK_ALIGNMENT,1);
		glPixelStorei(GL_PACK_ROW_LENGTH,_radDim.x);
		glPixelStorei(GL_PACK_SKIP_ROWS,0);
		glPixelStorei(GL_PACK_SKIP_PIXELS,0);

		glGenRenderbuffers(1,&_renderStencilBuffer);
		glBindRenderbuffer(GL_RENDERBUFFER,_renderStencilBuffer);
		glRenderbufferStorage(GL_RENDERBUFFER,GL_STENCIL_INDEX,_radDim.x,_radDim.y);

		glDrawPixels(_radDim.x,_radDim.y,GL_STENCIL_INDEX,GL_UNSIGNED_BYTE,stencilImg.data);

		// ...

		glReadPixels(0,0,_radDim.x,_radDim.y,GL_STENCIL_INDEX,GL_UNSIGNED_BYTE,stencilImgRead.data);
	}
	// end sample code from gamedev

		// copy the buffer back into the Bitmap, and then use the Bitmap to write it back to disk.
	memcpy(data.Scan0, buffer, size);
	image->UnlockBits(&data);

	CLSID clsId;
	int retVal = GetEncoderClsid(L"image/bmp", &clsId);
	image->Save(L"StencilGPU.bmp", &clsId, NULL);
	delete image;
	delete buffer;
}

//Define an error callback  
static void error_callback(int error, const char* description)  
{  
	fputs(description, stderr);  
	_fgetchar();  
}  

//Define the key input callback  
static void key_callback(GLFWwindow* window, int key, int scancode, int action, int mods)  
{  
	if (key == GLFW_KEY_ESCAPE && action == GLFW_PRESS)  
		glfwSetWindowShouldClose(window, GL_TRUE);  
}  

int main( void )  
{  
	//Set the error callback  
	glfwSetErrorCallback(error_callback);  

	//Initialize GLFW  
	if (!glfwInit())  
	{  
		exit(EXIT_FAILURE);  
	}  

	//Set the GLFW window creation hints - these are optional  
	//glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 3); //Request a specific OpenGL version  
	//glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 3); //Request a specific OpenGL version  
	//glfwWindowHint(GLFW_SAMPLES, 4); //Request 4x antialiasing  
	//glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE);  

	//Declare a window object  
	GLFWwindow* window;  

	//Create a window and create its OpenGL context  
	window = glfwCreateWindow(640, 480, "Test Window", NULL, NULL);  

	//If the window couldn't be created  
	if (!window)  
	{  
		fprintf( stderr, "Failed to open GLFW window.\n" );  
		glfwTerminate();  
		exit(EXIT_FAILURE);  
	}  

	//This function makes the context of the specified window current on the calling thread.   
	glfwMakeContextCurrent(window);  

	//Sets the key callback  
	glfwSetKeyCallback(window, key_callback);  

	//Initialize GLEW  
	GLenum err = glewInit();  

	//If GLEW hasn't initialized  
	if (err != GLEW_OK)   
	{  
		fprintf(stderr, "Error: %s\n", glewGetErrorString(err));  
		return -1;  
	}  

	//Set a background color  
	glClearColor(0.0f, 0.0f, 1.0f, 0.0f);  

	GdiplusStartupInput gdiplusStartupInput;
	ULONG_PTR gdiplusToken;
	GdiplusStartup(&gdiplusToken, &gdiplusStartupInput, NULL);

	TrapError();
	SetupStencilBuffer();

	//Main Loop  
	do  
	{  
		//Clear color buffer  
		glClear(GL_COLOR_BUFFER_BIT);  

		//Swap buffers  
		glfwSwapBuffers(window);  
		//Get and organize events, like keyboard and mouse input, window resizing, etc...  
		glfwPollEvents();  

	} //Check if the ESC key had been pressed or if the window had been closed  
	while (!glfwWindowShouldClose(window));  

	//Close OpenGL window and terminate GLFW  
	glfwDestroyWindow(window);  
	//Finalize and clean up GLFW  
	glfwTerminate();  

	GdiplusShutdown(gdiplusToken);

	exit(EXIT_SUCCESS);  
}  

Thank you very much for you help. I'll download your example and test it. I'm not sure why OpenCV would be a problem but I'll take your advice and look into it further. Thank you.