Picking Intersection Vertices

you should not get just the triangle, but also the point in the triangle. From there, just inspect the verticies of the triangle and find the closest of the triple.

Still having issues getting this to work, I tried to get the shortest distance from the mouse click but its not working right. Just want the closest vertex to be selected from click on the triangle

void PickingApp::pick(int sx, int sy)
{
	D3DXMATRIX P = GetCamera().proj();

	// Compute picking ray in view space.
	float vx = (+2.0f*sx/mClientWidth  - 1.0f)/P(0,0);
	float vy = (-2.0f*sy/mClientHeight + 1.0f)/P(1,1);

	D3DXVECTOR3 rayOrigin(0.0f, 0.0f, 0.0f);
	D3DXVECTOR3 rayDir(vx, vy, 1.0f);

	// Tranform to world space.
	D3DXMATRIX V = GetCamera().view();

	D3DXMATRIX inverseV;
	D3DXMatrixInverse(&inverseV, 0, &V);

	D3DXVec3TransformCoord(&rayOrigin, &rayOrigin, &inverseV);
	D3DXVec3TransformNormal(&rayDir, &rayDir, &inverseV);
	
	// Transform to the mesh's local space.
	D3DXMATRIX inverseW;
	D3DXMatrixInverse(&inverseW, 0, &mBaseWorld);

	D3DXVec3TransformCoord(&rayOrigin, &rayOrigin, &inverseW);
	D3DXVec3TransformNormal(&rayDir, &rayDir, &inverseW);
	
	ID3DX10Mesh* d3dxmesh = mBaseMesh.d3dxMesh();
	
	MouseCoord = rayOrigin;

	UINT hitCount;
	float u,v,t;
	ID3D10Blob* allHits;
	d3dxmesh->Intersect(&rayOrigin, &rayDir, &hitCount, 
		&mPickedTriangle, &u, &v, &t, &allHits);

	ReleaseCOM(allHits);
}

drawScene(){
...
...
DWORD i0 = indices[mPickedTriangle*3+0];
DWORD i1 = indices[mPickedTriangle*3+1];
DWORD i2 = indices[mPickedTriangle*3+2];

HR(ib->Unmap());

// Get the vertices of the picked triangle and copy them into our VB.

MeshVertex* vertices = 0;
HR(vb->Map((void**)&vertices, &size));

D3DXVECTOR3* triVerts = 0;
		
HR(mPickedTriVB->Map(D3D10_MAP_WRITE_DISCARD, 0, (void**)&triVerts));

float v1 = sqrt((pow(vertices[i0].pos.x - MouseCoord.x  ,2) + pow(vertices[i0].pos.y - MouseCoord.y,2) + pow(vertices[i0].pos.z - MouseCoord.z,2)));

float v2 = sqrt((pow(vertices[i1].pos.x - MouseCoord.x  ,2) + pow(vertices[i1].pos.y - MouseCoord.y,2) + pow(vertices[i1].pos.z - MouseCoord.z,2)));

float v3 = sqrt((pow(vertices[i2].pos.x - MouseCoord.x  ,2) + pow(vertices[i2].pos.y - MouseCoord.y,2) + pow(vertices[i2].pos.z - MouseCoord.z,2)));

		if(v1 < v2 && v1 < v3)
			triVerts[0] = vertices[i0].pos;
		if(v2 < v1 && v2 < v3)
			triVerts[0] = vertices[i1].pos;
		if(v3 < v1 && v3 < v2)
			triVerts[0] = vertices[i2].pos;
...}

Finally figured it out. Was mistaking RayOrgin for point in triangle clicked in its local space, but realized that no matter where I clicked the same spot would be selected. So I used the BaryCentric coordinates u,v to find highest influence, which results in the vertice that is the closest on the face to be selected. **This is from Frank D Luna directX book modified for point selection.

void PickingApp::pick(int sx, int sy)
{
	D3DXMATRIX P = GetCamera().proj();
	// Compute picking ray in view space.
	float vx = (+2.0f*sx/mClientWidth  - 1.0f)/P(0,0);
	float vy = (-2.0f*sy/mClientHeight + 1.0f)/P(1,1);

	D3DXVECTOR3 rayOrigin(0.0f, 0.0f, 0.0f);
	D3DXVECTOR3 rayDir(vx, vy, 1.0f);

	// Tranform to world space.
	D3DXMATRIX V = GetCamera().view();

	D3DXMATRIX inverseV;
	D3DXMatrixInverse(&inverseV, 0, &V);

	D3DXVec3TransformCoord(&rayOrigin, &rayOrigin, &inverseV);
	D3DXVec3TransformNormal(&rayDir, &rayDir, &inverseV);

	// Transform to the mesh's local space.
	D3DXMATRIX inverseW;
	D3DXMatrixInverse(&inverseW, 0, &mBaseWorld);

	D3DXVec3TransformCoord(&rayOrigin, &rayOrigin, &inverseW);
	D3DXVec3TransformNormal(&rayDir, &rayDir, &inverseW);
	
	ID3DX10Mesh* d3dxmesh = mBaseMesh.d3dxMesh();
	
	UINT hitCount;
	float u,v,t;
	ID3D10Blob* allHits;
	d3dxmesh->Intersect(&rayOrigin, &rayDir, &hitCount, 
		&mPickedTriangle, &u, &v, &t, &allHits);

	float v0 = 1 - u - v;
	float v1 = u;
	float v2 = v;

	if(v0 > v1 && v0 > v2)
		inf = mPickedTriangle * 3;
	if(v1 > v0 && v1 > v2)
		inf = mPickedTriangle * 3 + 1;
	if(v2 > v0 && v2 > v1)
		inf = mPickedTriangle * 3 + 2;

	ReleaseCOM(allHits);
}
drawScene(){
.....
......
if( mPickedTriangle != -1 )
    {
        ID3DX10Mesh* d3dxmesh = mBaseMesh.d3dxMesh();

        ID3DX10MeshBuffer* vb = 0;
        ID3DX10MeshBuffer* ib = 0;
        HR(d3dxmesh->GetVertexBuffer(0, &vb));
        HR(d3dxmesh->GetIndexBuffer(&ib));

        // Get the indices of the picked triangle.

        DWORD* indices = 0;
        SIZE_T size;
        HR(ib->Map((void**)&indices, &size));
        
        DWORD i0 = indices[inf];
    
        HR(ib->Unmap());

        // Get the vertices of the picked triangle and copy them into our VB.

        MeshVertex* vertices = 0;
        HR(vb->Map((void**)&vertices, &size));

        D3DXVECTOR3* triVerts = 0;
        
        HR(mPickedTriVB->Map(D3D10_MAP_WRITE_DISCARD, 0, (void**)&triVerts));
        
        triVerts[0] = vertices[i0].pos;
        
        mPickedTriVB->Unmap();
        HR(vb->Unmap());

        ReleaseCOM(vb);
        ReleaseCOM(ib);

        // Render the picked triangle specially to distinguish it
        // from the other triangles.

        md3dDevice->IASetInputLayout(InputLayout::Pos);
        md3dDevice->IASetPrimitiveTopology(D3D10_PRIMITIVE_TOPOLOGY_POINTLIST);
        UINT stride = sizeof(D3DXVECTOR3);
        UINT offset = 0;
        md3dDevice->IASetVertexBuffers(0, 1, &mPickedTriVB, &stride, &offset);

        D3D10_TECHNIQUE_DESC techDesc;
        mPickedTech->GetDesc( &techDesc );

        D3DXMATRIX W = mBaseWorld;
        D3DXMATRIX V = GetCamera().view();
        D3DXMATRIX P = GetCamera().proj();

        D3DXMATRIX WVP = W*V*P;

        mfxPickedWVPVar->SetMatrix((float*)&WVP);

        for(UINT i = 0; i < techDesc.Passes; ++i)
        {
            ID3D10EffectPass* pass = mPickedTech->GetPassByIndex(i);
            pass->Apply(0);

            md3dDevice->Draw(1, 0);
        }
    }
}

.