c++ - OpenGL ray OBB intersection

Question

I want to implement object picking in 3D so I have a Ray from a point on the screen towards the scene using glm::unproject method "it returns the Y flipped so I use its negative value", the following code success always when the object is centered on the world origin but with another object that is transformed and the camera moved or rotated it may success and may not, i simulated the ray and it is already intersect the object, all coordinates are in the world space.

bool Engine::IntersectBox(Ray& ray,BoundingBox* boundingBox,GLfloat& distance){
V3* v=boundingBox->getVertices();
glm::vec4 vec(v->x,v->y,v->z,1);
vec=boundingBox->getMatrix()*vec;
GLfloat minX=vec.x;
GLfloat minY=vec.y;
GLfloat minZ=vec.z;
GLfloat maxX=vec.x;
GLfloat maxY=vec.y;
GLfloat maxZ=vec.z;
for(int i=0;i<8;i++){
    v++;
    vec=glm::vec4(v->x,v->y,v->z,1);
    vec=boundingBox->getMatrix()*vec;
    minX=minX<vec.x?minX:vec.x;
    minY=minY<vec.y?minY:vec.y;
    minZ=minZ<vec.z?minZ:vec.z;
    maxX=maxX>vec.x?maxX:vec.x;
    maxY=maxY>vec.y?maxY:vec.y;
    maxZ=maxZ>vec.z?maxZ:vec.z;
}
GLfloat tMin = 0.0f;
GLfloat tMax = 100000.0f;
glm::vec3 delta=glm::vec3(boundingBox->getMatrix()[3])-ray.getOrigin();
{
    glm::vec3 xAxis=boundingBox->getMatrix()[0];
    GLfloat e = glm::dot(xAxis, delta);
    GLfloat f = glm::dot(ray.getDirection(), xAxis);
    if ( fabs(f) > 0.001f ) { // Standard case
        GLfloat min = (e+minX)/f; // Intersection with the "left" plane
        GLfloat max = (e+maxX)/f; // Intersection with the "right" plane
        if(min<max){
            tMin=min;
            tMax=max;
        }
        else{
            tMin=max;
            tMax=min;
        }
        if (tMax < tMin)
            return false;
    }
    else{
        if(-e+minX > 0.0f || -e+maxX < 0.0f)
            return false;
    }
}
{
    glm::vec3 yAxis=boundingBox->getMatrix()[1];
    GLfloat e = glm::dot(yAxis, delta);
    GLfloat f = glm::dot(ray.getDirection(), yAxis);
    if ( fabs(f) > 0.001f ){
        GLfloat min = (e+minY)/f;
        GLfloat max = (e+maxY)/f;
        if(min<max){
            tMin=glm::max(tMin,min);
            tMax=glm::min(tMax,max);
        }
        else{
            tMin=glm::max(tMin,max);
            tMax=glm::min(tMax,min);
        }
        if (tMax < tMin)
            return false;
    }else{
        if(-e+minY > 0.0f || -e+maxY < 0.0f)
            return false;
    }
}
{
    glm::vec3 zAxis=boundingBox->getMatrix()[2];
    GLfloat e = glm::dot(zAxis, delta);
    GLfloat f = glm::dot(ray.getDirection(),zAxis);
    if ( fabs(f) > 0.001f ){
        GLfloat min = (e+minZ)/f;
        GLfloat max = (e+maxZ)/f;
        if(min<max){
            tMin=glm::max(tMin,min);
            tMax=glm::min(tMax,max);
        }
        else{
            tMin=glm::max(tMin,max);
            tMax=glm::min(tMax,min);
        }
        if (tMax < tMin)
            return false;
    }else{
        if(-e+minZ > 0.0f || -e+maxZ < 0.0f)
            return false;
    }
}
distance = tMin;
return true;
}

Answer 1

I am doing this using:

• OpenGL 3D-raypicking with high poly meshes

The idea is to apart of rendering to screen also render index of each object into separate unseen buffer (color attachment, stencil, shadow,...) and than just pick pixel at mouse position from this buffer and depth ... which provides 3D position of the picked point and also index of object that it belongs to. This is very fast O(1) at almost no performance cost.

Now You do not need OBB for your objects nor any intersection checking anymore. Instead have a local coordinate system in form of 4x4 homogenuous matrix with which you can easily convert the 3D position picked by mouse into object local coordinates making the manipulation like translation/rotation of the object really easy.

Here is my older C++ approach of mine for this:

• Compute objects moving with arrows and mouse

which does not require any additional libs and stuff. How ever I do it now using all above in fusion like this:

//---------------------------------------------------------------------------
#ifndef _OpenGLctrl3D_h
#define _OpenGLctrl3D_h
//---------------------------------------------------------------------------
#include "gl/OpenGL3D_double.cpp" // vector and matrix math keyboard and mouse handler
//---------------------------------------------------------------------------
static reper NULL_rep;
AnsiString dbg="";
//---------------------------------------------------------------------------
class OpenGLctrl3D      // arrow translation controls (you need one for each objet)
    {
public:
    reper *rep;             // points to bounded object model matrix
    double l[3],r0,r1,r2,a; // l  - size of each straight arrow
                            // r0 - tube radius
                            // r1 - arrow radius
                            // r2 - arced arrow radius
                            // a  - arrowhead size
    double a0,a1,aa;        // start,end, cone size [rad] of the arced arrow

    OpenGLctrl3D()
        {
        rep=&NULL_rep;
        l[0]=3.5; r0=0.05; a0=  0.0*deg; a=0.10;
        l[1]=3.5; r1=0.25; a1=360.0*deg;
        l[2]=3.5; r2=0.50; aa= 15.0*deg;
        }
    OpenGLctrl3D(OpenGLctrl3D& a)   { *this=a; }
    ~OpenGLctrl3D() {}
    OpenGLctrl3D* operator = (const OpenGLctrl3D *a) { *this=*a; return this; }
    //OpenGLctrl3D* operator = (const OpenGLctrl3D &a) { ...copy... return this; }

    void draw(int sel);                 // render arrows
    void mouse_select(void* sys);       // handle [camera local] mouse events (no active button)
    void mouse_edit  (void* sys);       // handle [camera local] mouse events (active button)
    };
//---------------------------------------------------------------------------
class OpenGLctrls3D     // arrow translation controls (you need one for each objet)
    {
public:
    reper *eye;                         // camera matrix
    double per[16],ndc[16];             // perspective and viewport matrices
    TShiftState sh; double mw[3],ms[3]; // actual mouse [buttons],[world units],[camera units]
    bool _redraw;                       // redraw needed?
    int sel0,sel1,_sel;                 // actualy selected item ctrl[sel0].axis=sel1 the _sel is for iteration variable
    double psel[3];                     // selected point [object local units]

    List<OpenGLctrl3D> ctrl;
    OpenGLctrls3D() { eye=&NULL_rep; matrix_one(per); matrix_one(ndc); ctrl.num=0; }
    OpenGLctrls3D(OpenGLctrls3D& a) { *this=a; }
    ~OpenGLctrls3D(){}
    OpenGLctrls3D* operator = (const OpenGLctrls3D *a) { *this=*a; return this; }
    //OpenGLctrls3D* operator = (const OpenGLctrls3D &a) { ...copy... return this; }
    void add(reper &rep,double *l,double r0,double r1,double r2,double a)   // add new control bounded to rep
        {
        // l  - size of each straight arrow
        // r0 - tube radius
        // r1 - arrow radius
        // r2 - arced arrow radius
        // a  - arrowhead size
        ctrl.add();
        OpenGLctrl3D *c=ctrl.dat+ctrl.num-1;
        c->rep=&rep;
        vector_copy(c->l,l);
        c->r0=r0;
        c->r1=r1;
        c->r2=r2;
        c->a=a;
        }
    void resize(int x0,int y0,int xs,int ys)
        {
        matrix_one(ndc);
        ndc[ 0]=+divide(2.0,double(xs));
        ndc[ 5]=-divide(2.0,double(ys));
        ndc[12]=-1.0;
        ndc[13]=+1.0;
        glGetDoublev(GL_PROJECTION_MATRIX,per);
        mouse_refresh();
        }
    void draw()
        {
        int i;
        OpenGLctrl3D *c;
        for (c=ctrl.dat,i=0;i<ctrl.num;i++,c++)
            {
            glPushMatrix();
            c->rep->use_rep();
            glMatrixMode(GL_MODELVIEW);
            glMultMatrixd(c->rep->rep);
            if (i==sel0) c->draw(sel1);
             else        c->draw(-1);
            glMatrixMode(GL_MODELVIEW);
            glPopMatrix();
            }
        }
    bool mouse(double mx,double my,TShiftState _sh) // handle mouse events return if redraw is needed
        {
        // mouse depth [camera units]
        ms[0]=mx; ms[1]=my; sh=_sh;
        ms[2]=glReadDepth(mx,divide(-2.0,ndc[5])-my-1,per);
        // mouse x,y [pixel] ->  <-1,+1> NDC
        matrix_mul_vector(ms,ndc,ms);
        // mouse x,y <-1,+1> NDC -> [camera units]
        scr2world(mw,ms);
        return mouse_refresh();
        }
    bool mouse_refresh()    // call after any view change
        {
        _redraw=false;
        if (!sh.Contains(ssLeft))
            {
            int _sel0=sel0; sel0=-1;
            int _sel1=sel1; sel1=-1;
            for (_sel=0;_sel<ctrl.num;_sel++) ctrl.dat[_sel].mouse_select(this);
            _redraw=((_sel0!=sel0)||(_sel1!=sel1));
            }
        else{
            if ((sel0>=0)&&(sel0<ctrl.num)) ctrl.dat[sel0].mouse_edit(this);
            }
        return _redraw;
        }
    void world2scr(double *s,double *w)
        {
        // camera [LCS]
        eye->g2l(s,w);
        // [camera units] -> <-1,+1> NDC
        s[0]=-divide(s[0]*per[0],s[2]);
        s[1]=-divide(s[1]*per[5],s[2]);
        }
    void scr2world(double *w,double *s)
        {
        // <-1,+1> NDC -> [camera units]
        w[0]=-divide(s[0]*s[2],per[0]);
        w[1]=-divide(s[1]*s[2],per[5]);
        w[2]=s[2];
        // world [GCS]
        eye->l2g(w,w);
        }
    };
//---------------------------------------------------------------------------
//---------------------------------------------------------------------------
void OpenGLctrl3D::draw(int sel)
    {
    if (sel==0) glColor3f(1.0,0.0,0.0); else glColor3f(0.5,0.0,0.0); glArrowx(0.0,0.0,0.0,r0,r1,l[0],a);
    if (sel==1) glColor3f(0.0,1.0,0.0); else glColor3f(0.0,0.5,0.0); glArrowy(0.0,0.0,0.0,r0,r1,l[1],a);
    if (sel==2) glColor3f(0.0,0.0,1.0); else glColor3f(0.0,0.0,0.5); glArrowz(0.0,0.0,0.0,r0,r1,l[2],a);
    if (sel==3) glColor3f(1.0,0.0,0.0); else glColor3f(0.5,0.0,0.0); glCircleArrowyz(0.0,0.0,0.0,r2,r0,r1,a0,a1,aa);
    if (sel==4) glColor3f(0.0,1.0,0.0); else glColor3f(0.0,0.5,0.0); glCircleArrowzx(0.0,0.0,0.0,r2,r0,r1,a0,a1,aa);
    if (sel==5) glColor3f(0.0,0.0,1.0); else glColor3f(0.0,0.0,0.5); glCircleArrowxy(0.0,0.0,0.0,r2,r0,r1,a0,a1,aa);
    }
//---------------------------------------------------------------------------
void OpenGLctrl3D::mouse_select(void *_sys)
    {
    OpenGLctrls3D *sys=(OpenGLctrls3D*)_sys;
    int i,x,y,z; double p[3],q[3],pm[3],t,r;
    // mouse [object local units]
    rep->g2l(pm,sys->mw);
    // straight arrows
    for (i=0;i<3;i++)
        {
        t=pm[i]; pm[i]=0.0; r=vector_len(pm); pm[i]=t;
        t=divide(l[i]-t,a);
        if ((t>=0.0)&&(t<=1.0)&&(r<=r1*t))  // straight cone
            {
            sys->sel0=sys->_sel;
            sys->sel1=i;
            vector_ld(sys->psel,0.0,0.0,0.0); sys->psel[i]=pm[i];
            }
        }
    // arced arrows
    for (i=0;i<3;i++)
        {
        if (i==0){ x=1; y=2; z=0; }
        if (i==1){ x=2; y=0; z=1; }
        if (i==2){ x=0; y=1; z=2; }
        t=atanxy(pm[x],pm[y]);
        p[x]=r2*cos(t);
        p[y]=r2*sin(t);
        p[z]=0.0;
        vector_sub(q,p,pm);
        r=vector_len(q);
        if (r<=r0*2.0)
            {
            sys->sel0=sys->_sel;
            sys->sel1=i+3;
            vector_copy(sys->psel,p);
            }
        }
    }
//---------------------------------------------------------------------------
void OpenGLctrl3D::mouse_edit(void *_sys)
    {
    OpenGLctrls3D *sys=(OpenGLctrls3D*)_sys;
    // drag straight arrows (active button)
    if ((sys->sel1>=0)&&(sys->sel1<3))
        {
        double z0,z1,z2,t0;
        double q[3],q0[3],q1[3],t;
        // q0 = mouse change in 2D screen space
        rep->l2g(q0,sys->psel);                 // selected point position
        sys->world2scr(q0,q0);
        vector_sub(q0,q0,sys->ms); q0[2]=0.0;   // actual mouse position
        // q1 = selected axis step in 2D screen space
        rep->l2g(q,sys->psel);                  // selected point position
        sys->world2scr(q,q);
        vector_copy(q1,sys->psel);              // axis step
        q1[sys->sel1]+=1.0;
        rep->l2g(q1,q1);
        sys->world2scr(q1,q1);
        vector_sub(q1,q1,q); q1[2]=0.0;
        // compute approx change
        t=-vector_mul(q0,q1);                   // dot(q0,q1)
        // enhance precision of t
        int i; double len0,len,dq[3]={0.0,0.0,0.0},dt;
        // selected arrow direction
        dq[sys->sel1]=1.0;
        // closest point on axis to psel
        for (len0=-1.0,dt=0.25*t;fabs(dt)>1e-5;t+=dt)
            {
            // position on axis p(t) = p0 + t*dp
            for (i=0;i<3;i++) q[i]=sys->psel[i]+(t*dq[i]);
            // len = distance to mouse
            rep->l2g(q,q);
            sys->world2scr(q,q);
            vector_sub(q,q,sys->ms); q[2]=0.0;
            len=vector_len2(q);
            // handle iteration step
            if (len0<-0.5) len0=len;
            if (len>len0) dt=-0.1*dt;
            len0=len;
            }
        // translate by change
        double m[16]=
            {
            1.0,0.0,0.0,0.0,
            0.0,1.0,0.0,0.0,
            0.0,0.0,1.0,0.0,
            0.0,0.0,0.0,1.0,
            };
        m[12+sys->sel1]=t;
        rep->use_rep();
        matrix_mul(rep->rep,m,rep->rep);
        rep->_inv=0;
        sys->_redraw=true;
        }
    // rotate arced arrows (active button)
    if ((sys->sel1>=3)&&(sys->sel1<6))
        {
        int i,x,y,z; double t,t0,tt,dt,len,len0,q[3];
        if (sys->sel1==3){ x=1; y=2; z=0; }
        if (sys->sel1==4){ x=2; y=0; z=1; }
        if (sys->sel1==5){ x=0; y=1; z=2; }
        t0=atanxy(sys->psel[x],sys->psel[y]);
        // initial search
        for (i=10,t=0.0,dt=divide(1.0,i),len0=-1.0;i--;t+=dt)
            {
            q[x]=r2*cos(t0+t);
            q[y]=r2*sin(t0+t);
            q[z]=0.0;
            rep->l2g(q,q);
            sys->world2scr(q,q);
            vector_sub(q,q,sys->ms); q[2]=0.0;
            len