Setting OpenCV to access data from Kinect in Linux

This post will help you to access Microsoft Kinect data using libfreenect driver from OpenKinect in OpenCV. The method is tested and verified in our lab running PC with Ubuntu 12.04 LTS where OpenCV is installed from Ubuntu Software centre and with Kinect for Windows hardware. OpenKinect   Prerequisites 1) OpenCV 2.3+ 2) GCC compiler First we need to install libfreenect – a linux based driver(can also be used in windows). Download the source from the github channel from this link. Open a terminal and install some programs required for compiling and installing libfreenect as below.

sudo apt-get install git-core cmake pkg-config build-essential libusb-1.0-0-dev
sudo apt-get install libglut3-dev libxmu-dev libxi-dev

After installing the necessary components use the following commands to build and install libreenect in your PC.

cd libfreenect-stable
mkdir build
cd build
cmake -L .. -DBUILD_AUDIO=ON -DBUILD_CV=ON
make

Edit the generated cmake file for installing to /usr instead of /usr/local which is the default configuration. For this purpose open cmake_install.cmake file in the build directory and change the line SET(CMAKE_INSTALL_PREFIX “/usr/local”) to SET(CMAKE_INSTALL_PREFIX “/usr”) Then type the following in a terminal to install the libfreenect.

sudo make install

This will install the libfreenect to your computer. Next we need to configure eclipse to work with libfreenect and OpenCV libraries. Create an empty C++ project and select Linux GCC compiler for compilation. Create a source file named main.cpp and copy the following code to it.

#include "libfreenect/libfreenect.hpp"
#include <iostream>
#include <vector>
#include <cmath>
#include <pthread.h>
#include <opencv2/opencv.hpp>

using namespace cv;
using namespace std;


class myMutex {
 public:
 myMutex() {
 pthread_mutex_init( &m_mutex, NULL );
 }
 void lock() {
 pthread_mutex_lock( &m_mutex );
 }
 void unlock() {
 pthread_mutex_unlock( &m_mutex );
 }
 private:
 pthread_mutex_t m_mutex;
};


class MyFreenectDevice : public Freenect::FreenectDevice {
 public:
 MyFreenectDevice(freenect_context *_ctx, int _index)
 : Freenect::FreenectDevice(_ctx, _index), m_buffer_depth(FREENECT_DEPTH_11BIT),
 m_buffer_rgb(FREENECT_VIDEO_RGB), m_gamma(2048), m_new_rgb_frame(false),
 m_new_depth_frame(false), depthMat(Size(640,480),CV_16UC1),
 rgbMat(Size(640,480), CV_8UC3, Scalar(0)),
 ownMat(Size(640,480),CV_8UC3,Scalar(0)) {

 for( unsigned int i = 0 ; i < 2048 ; i++) {
 float v = i/2048.0;
 v = std::pow(v, 3)* 6;
 m_gamma[i] = v*6*256;
 }
 }

 // Do not call directly even in child
 void VideoCallback(void* _rgb, uint32_t timestamp) {
 std::cout << "RGB callback" << std::endl;
 m_rgb_mutex.lock();
 uint8_t* rgb = static_cast<uint8_t*>(_rgb);
 rgbMat.data = rgb;
 m_new_rgb_frame = true;
 m_rgb_mutex.unlock();
 };

 // Do not call directly even in child
 void DepthCallback(void* _depth, uint32_t timestamp) {
 std::cout << "Depth callback" << std::endl;
 m_depth_mutex.lock();
 uint16_t* depth = static_cast<uint16_t*>(_depth);
 depthMat.data = (uchar*) depth;
 m_new_depth_frame = true;
 m_depth_mutex.unlock();
 }

 bool getVideo(Mat& output) {
 m_rgb_mutex.lock();
 if(m_new_rgb_frame) {
 cv::cvtColor(rgbMat, output, CV_RGB2BGR);
 m_new_rgb_frame = false;
 m_rgb_mutex.unlock();
 return true;
 } else {
 m_rgb_mutex.unlock();
 return false;
 }
 }

 bool getDepth(Mat& output) {
 m_depth_mutex.lock();
 if(m_new_depth_frame) {
 depthMat.copyTo(output);
 m_new_depth_frame = false;
 m_depth_mutex.unlock();
 return true;
 } else {
 m_depth_mutex.unlock();
 return false;
 }
 }
 private:
 std::vector<uint8_t> m_buffer_depth;
 std::vector<uint8_t> m_buffer_rgb;
 std::vector<uint16_t> m_gamma;
 Mat depthMat;
 Mat rgbMat;
 Mat ownMat;
 myMutex m_rgb_mutex;
 myMutex m_depth_mutex;
 bool m_new_rgb_frame;
 bool m_new_depth_frame;
};


int main(int argc, char **argv) {
 bool die(false);
 string filename("snapshot");
 string suffix(".png");
 int i_snap(0),iter(0);

 Mat depthMat(Size(640,480),CV_16UC1);
 Mat depthf (Size(640,480),CV_8UC1);
 Mat rgbMat(Size(640,480),CV_8UC3,Scalar(0));
 Mat ownMat(Size(640,480),CV_8UC3,Scalar(0));

 // The next two lines must be changed as Freenect::Freenect
 // isn't a template but the method createDevice:
 // Freenect::Freenect<MyFreenectDevice> freenect;
 // MyFreenectDevice& device = freenect.createDevice(0);
 // by these two lines:

 Freenect::Freenect freenect;
 MyFreenectDevice& device = freenect.createDevice<MyFreenectDevice>(0);

 namedWindow("rgb",CV_WINDOW_AUTOSIZE);
 namedWindow("depth",CV_WINDOW_AUTOSIZE);
 device.startVideo();
 device.startDepth();
 while (!die) {
 device.getVideo(rgbMat);
 device.getDepth(depthMat);
 cv::imshow("rgb", rgbMat);
 depthMat.convertTo(depthf, CV_8UC1, 255.0/2048.0);
 cv::imshow("depth",depthf);
 char k = cvWaitKey(5);
 if( k == 27 ){
 cvDestroyWindow("rgb");
 cvDestroyWindow("depth");
 break;
 }
 if( k == 8 ) {
 std::ostringstream file;
 file << filename << i_snap << suffix;
 cv::imwrite(file.str(),rgbMat);
 i_snap++;
 }
 if(iter >= 1000) break;
 iter++;
 }

 device.stopVideo();
 device.stopDepth();
 return 0;
}

Add these to the library options in eclipse as shown in the below figure freenect, freenect_cv, freenect_sync, opencv_core, opencv_imgproc, opencv_highgui Libfreenect Kinect Opencv Add the following directory in the Includes tab under GCC C++ Compiler  /usr/include/libusb-1.0 and press OK.   Compile the code and run. Happy Kinecting!!

Advertisements

9 thoughts on “Setting OpenCV to access data from Kinect in Linux

  1. help me , its giving me this error:
    /usr/local/include/libfreenect/libfreenect.hpp:33:20: fatal error: libusb.h: No such file or directory.

    • go find the libfreenect.hpp in 33:20, change #include to #include , you need sudo vim to help you with this. and try locate libusb.h

  2. Hello, I added the libraries and everything but the compiler is giving me this error:
    ./openKinect_main.o: undefined reference to symbol ‘pthread_create@@GLIBC_2.2.5’
    did I miss something? Help me please. Thanks

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s