Visualization of the NARF descriptor and descriptor distances

This tutorial is about the visualization of how the NARF descriptor is calculated and to test how the descriptor distances between certain points in a range image behave. Compared to the other tuturials, this one is not really about the code, but about trying the program and looking at the visualization. Of course, nothing keeps you from having a look at it anyway.

The code

First, create a file called, let’s say, narf_descriptor_visualization.cpp in your favorite editor, and place the following code inside it:

  1/* \author Bastian Steder */
  2
  3#include <iostream>
  4
  5#include <pcl/point_cloud.h>
  6#include <pcl/io/pcd_io.h>
  7#include <pcl/visualization/range_image_visualizer.h>
  8#include <pcl/range_image/range_image.h>
  9#include <pcl/features/narf.h>
 10#include <pcl/console/parse.h>
 11#include <pcl/common/file_io.h> // for getFilenameWithoutExtension
 12
 13float angular_resolution = 0.5f;
 14int rotation_invariant = 0;
 15float support_size = 0.3f;
 16int descriptor_size = 36;
 17pcl::RangeImage::CoordinateFrame coordinate_frame = pcl::RangeImage::CAMERA_FRAME;
 18bool setUnseenToMaxRange = false;
 19
 20typedef pcl::PointXYZ PointType;
 21
 22void 
 23printUsage (const char* progName)
 24{
 25  std::cout << "\n\nUsage: "<<progName<<" [options] <scene.pcd>\n\n"
 26            << "Options:\n"
 27            << "-------------------------------------------\n"
 28            << "-r <float>   angular resolution in degrees (default "<<angular_resolution<<")\n"
 29            << "-s <float>   support size for the interest points (diameter of the used sphere - "
 30            <<                                                     "default "<<support_size<<")\n"
 31            << "-d <int>     descriptor size (default "<<descriptor_size<<")\n"
 32            << "-c <int>     coordinate frame of the input point cloud (default "<< (int)coordinate_frame<<")\n"
 33            << "-o <0/1>     switch rotational invariant version of the feature on/off"
 34            <<               " (default "<< (int)rotation_invariant<<")\n"
 35            << "-m           set unseen pixels to max range\n"
 36            << "-h           this help\n"
 37            << "\n\n";
 38}
 39
 40int 
 41main (int argc, char** argv)
 42{
 43  // --------------------------------------
 44  // -----Parse Command Line Arguments-----
 45  // --------------------------------------
 46  if (pcl::console::find_argument (argc, argv, "-h") >= 0)
 47  {
 48    printUsage (argv[0]);
 49    return 0;
 50  }
 51  if (pcl::console::find_argument (argc, argv, "-m") >= 0)
 52  {
 53    setUnseenToMaxRange = true;
 54    std::cout << "Setting unseen values in range image to maximum range readings.\n";
 55  }
 56  if (pcl::console::parse (argc, argv, "-o", rotation_invariant) >= 0)
 57    std::cout << "Switching rotation invariant feature version "<< (rotation_invariant ? "on" : "off")<<".\n";
 58  int tmp_coordinate_frame;
 59  if (pcl::console::parse (argc, argv, "-c", tmp_coordinate_frame) >= 0)
 60  {
 61    coordinate_frame = pcl::RangeImage::CoordinateFrame (tmp_coordinate_frame);
 62    std::cout << "Using coordinate frame "<< (int)coordinate_frame<<".\n";
 63  }
 64  if (pcl::console::parse (argc, argv, "-s", support_size) >= 0)
 65    std::cout << "Setting support size to "<<support_size<<".\n";
 66  if (pcl::console::parse (argc, argv, "-d", descriptor_size) >= 0)
 67    std::cout << "Setting descriptor size to "<<descriptor_size<<".\n";
 68  if (pcl::console::parse (argc, argv, "-r", angular_resolution) >= 0)
 69    std::cout << "Setting angular resolution to "<<angular_resolution<<"deg.\n";
 70  angular_resolution = pcl::deg2rad (angular_resolution);
 71  
 72
 73  // -----------------------
 74  // -----Read pcd file-----
 75  // -----------------------
 76  pcl::PointCloud<PointType>::Ptr point_cloud_ptr (new pcl::PointCloud<PointType>);
 77  pcl::PointCloud<PointType>& point_cloud = *point_cloud_ptr;
 78  pcl::PointCloud<pcl::PointWithViewpoint> far_ranges;
 79  Eigen::Affine3f scene_sensor_pose (Eigen::Affine3f::Identity ());
 80  std::vector<int> pcd_filename_indices = pcl::console::parse_file_extension_argument (argc, argv, "pcd");
 81  if (!pcd_filename_indices.empty ())
 82  {
 83    std::string filename = argv[pcd_filename_indices[0]];
 84    if (pcl::io::loadPCDFile (filename, point_cloud) == -1)
 85    {
 86      std::cout << "Was not able to open file \""<<filename<<"\".\n";
 87      printUsage (argv[0]);
 88      return 0;
 89    }
 90    scene_sensor_pose = Eigen::Affine3f (Eigen::Translation3f (point_cloud.sensor_origin_[0],
 91                                                               point_cloud.sensor_origin_[1],
 92                                                               point_cloud.sensor_origin_[2])) *
 93                        Eigen::Affine3f (point_cloud.sensor_orientation_);
 94    std::string far_ranges_filename = pcl::getFilenameWithoutExtension (filename)+"_far_ranges.pcd";
 95    if (pcl::io::loadPCDFile (far_ranges_filename.c_str (), far_ranges) == -1)
 96      std::cout << "Far ranges file \""<<far_ranges_filename<<"\" does not exists.\n";
 97  }
 98  else
 99  {
100    std::cout << "\nNo *.pcd file for scene given.\n\n";
101    printUsage (argv[0]);
102    return 1;
103  }
104  
105  // -----------------------------------------------
106  // -----Create RangeImage from the PointCloud-----
107  // -----------------------------------------------
108  float noise_level = 0.0;
109  float min_range = 0.0f;
110  int border_size = 1;
111  pcl::RangeImage::Ptr range_image_ptr (new pcl::RangeImage);
112  pcl::RangeImage& range_image = *range_image_ptr;
113  range_image.createFromPointCloud (point_cloud, angular_resolution, pcl::deg2rad (360.0f), pcl::deg2rad (180.0f),
114                                   scene_sensor_pose, coordinate_frame, noise_level, min_range, border_size);
115  range_image.integrateFarRanges (far_ranges);
116  if (setUnseenToMaxRange)
117    range_image.setUnseenToMaxRange ();
118  
119  // Extract NARF features:
120  std::cout << "Now extracting NARFs in every image point.\n";
121  std::vector<std::vector<pcl::Narf*> > narfs;
122  narfs.resize (range_image.size ());
123  unsigned int last_percentage=0;
124  for (unsigned int y=0; y<range_image.height; ++y)
125  {
126    for (unsigned int x=0; x<range_image.width; ++x)
127    {
128      const auto index = y*range_image.width+x;
129      const auto percentage = ((100*index) / range_image.size ());
130      if (percentage > last_percentage)
131      {
132        std::cout << percentage<<"% "<<std::flush;
133        last_percentage = percentage;
134      }
135      pcl::Narf::extractFromRangeImageAndAddToList (range_image, x, y, descriptor_size,
136                                                    support_size, rotation_invariant != 0, narfs[index]);
137      //std::cout << "Extracted "<<narfs[index].size ()<<" features for pixel "<<x<<","<<y<<".\n";
138    }
139  }
140  std::cout << "100%\n";
141  std::cout << "Done.\n\n Now you can click on points in the image to visualize how the descriptor is "
142       << "extracted and see the descriptor distances to every other point..\n";
143  
144  //---------------------
145  // -----Show image-----
146  // --------------------
147  pcl::visualization::RangeImageVisualizer range_image_widget ("Scene range image"),
148                                           surface_patch_widget("Descriptor's surface patch"),
149                                           descriptor_widget("Descriptor"),
150                                           descriptor_distances_widget("descriptor distances");
151  range_image_widget.showRangeImage (range_image);
152  //range_image_widget.visualize_selected_point = true;
153
154  //--------------------
155  // -----Main loop-----
156  //--------------------
157  while (true) 
158  {
159    range_image_widget.spinOnce ();  // process GUI events
160    surface_patch_widget.spinOnce ();  // process GUI events
161    descriptor_widget.spinOnce ();  // process GUI events
162    pcl_sleep(0.01);
163    
164    //if (!range_image_widget.mouse_click_happened)
165      continue;
166    //range_image_widget.mouse_click_happened = false;
167    //float clicked_pixel_x_f = range_image_widget.last_clicked_point_x,
168          //clicked_pixel_y_f = range_image_widget.last_clicked_point_y;
169    int clicked_pixel_x, clicked_pixel_y;
170    //range_image.real2DToInt2D (clicked_pixel_x_f, clicked_pixel_y_f, clicked_pixel_x, clicked_pixel_y);
171    if (!range_image.isValid (clicked_pixel_x, clicked_pixel_y))
172      continue;
173      //Vector3f clicked_3d_point;
174      //range_image.getPoint (clicked_pixel_x, clicked_pixel_y, clicked_3d_point);
175    
176    //surface_patch_widget.show (false);
177    //descriptor_widget.show (false);"
178    
179    int selected_index = clicked_pixel_y*range_image.width + clicked_pixel_x;
180    pcl::Narf narf;
181    if (!narf.extractFromRangeImage (range_image, clicked_pixel_x, clicked_pixel_y,
182                                                                         descriptor_size, support_size))
183    {
184      continue;
185    }
186    
187    int surface_patch_pixel_size = narf.getSurfacePatchPixelSize ();
188    float surface_patch_world_size = narf.getSurfacePatchWorldSize ();
189    surface_patch_widget.showFloatImage (narf.getSurfacePatch (), surface_patch_pixel_size, surface_patch_pixel_size,
190                                         -0.5f*surface_patch_world_size, 0.5f*surface_patch_world_size, true);
191    /*float surface_patch_rotation = narf.getSurfacePatchRotation ();
192    float patch_middle = 0.5f* (float (surface_patch_pixel_size-1));
193    float angle_step_size = pcl::deg2rad (360.0f)/narf.getDescriptorSize ();
194    float cell_size = surface_patch_world_size/float (surface_patch_pixel_size),
195          cell_factor = 1.0f/cell_size,
196          max_dist = 0.5f*surface_patch_world_size,
197          line_length = cell_factor* (max_dist-0.5f*cell_size);
198    for (int descriptor_value_idx=0; descriptor_value_idx<narf.getDescriptorSize (); ++descriptor_value_idx)
199    {
200      float angle = descriptor_value_idx*angle_step_size + surface_patch_rotation;
201      //surface_patch_widget.markLine (patch_middle, patch_middle, patch_middle+line_length*sinf (angle),
202                                     //patch_middle+line_length*-std::cos (angle), pcl::visualization::Vector3ub (0,255,0));
203    }
204    std::vector<float> rotations, strengths;
205    narf.getRotations (rotations, strengths);
206    float radius = 0.5f*surface_patch_pixel_size;
207    for (unsigned int i=0; i<rotations.size (); ++i)
208    {
209      //surface_patch_widget.markLine (radius-0.5, radius-0.5, radius-0.5f + 2.0f*radius*sinf (rotations[i]),
210                                                //radius-0.5f - 2.0f*radius*std::cos (rotations[i]), pcl::visualization::Vector3ub (255,0,0));
211    }*/
212    
213    descriptor_widget.showFloatImage (narf.getDescriptor (), narf.getDescriptorSize (), 1, -0.1f, 0.3f, true);
214
215    //===================================
216    //=====Compare with all features=====
217    //===================================
218    const std::vector<pcl::Narf*>& narfs_of_selected_point = narfs[selected_index];
219    if (narfs_of_selected_point.empty ())
220      continue;
221    
222    //descriptor_distances_widget.show (false);
223    float* descriptor_distance_image = new float[range_image.size ()];
224    for (unsigned int point_index=0; point_index<range_image.size (); ++point_index)
225    {
226      float& descriptor_distance = descriptor_distance_image[point_index];
227      descriptor_distance = std::numeric_limits<float>::infinity ();
228      std::vector<pcl::Narf*>& narfs_of_current_point = narfs[point_index];
229      if (narfs_of_current_point.empty ())
230        continue;
231      for (unsigned int i=0; i<narfs_of_selected_point.size (); ++i)
232      {
233        for (unsigned int j=0; j<narfs_of_current_point.size (); ++j)
234        {
235          descriptor_distance = (std::min)(descriptor_distance,
236                                           narfs_of_selected_point[i]->getDescriptorDistance (*narfs_of_current_point[j]));
237        }
238      }
239    }
240    descriptor_distances_widget.showFloatImage (descriptor_distance_image, range_image.width, range_image.height,
241                                               -std::numeric_limits<float>::infinity (), std::numeric_limits<float>::infinity (), true);
242    delete[] descriptor_distance_image;
243  }
244}

Compiling and running the program

Add the following lines to your CMakeLists.txt file:

 1cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
 2
 3project(narf_descriptor_visualization)
 4
 5find_package(PCL 1.3 REQUIRED)
 6
 7include_directories(${PCL_INCLUDE_DIRS})
 8link_directories(${PCL_LIBRARY_DIRS})
 9add_definitions(${PCL_DEFINITIONS})
10
11add_executable (narf_descriptor_visualization narf_descriptor_visualization.cpp)
12target_link_libraries (narf_descriptor_visualization ${PCL_LIBRARIES})

You can now try it with a point cloud file from your hard drive:

$ ./narf_descriptor_visualization <point_cloud.pcd>

It will take a few second, during which you will see the status in the terminal. During this time, a NARF feature is extracted in every point of the range image created from the given point cloud. When it is done, a widget showing the range image pops up. Now click on a point in the range image. If it is a valid image point, three additional widgets will pop up. One visualizing the actual descriptor as a row of gray values, one showing a local range image patch of the area on which you clicked, overlaid with a star shaped pattern. Each beam corresponds to one of the cells in the descriptor. The one facing upwards to the first cell and then going clockwise. The basic intuition is, that the more the surface changes under the beam, the higher (brighter) the value of the corresponding descriptor cell. There is also one or more red beams, which mark the extracted dominant orientations of the image patch, which, together with the normal, is used to create a unique orientation for the feature coordinate frame. The last image visualizes the descriptor distances to every other point in the scene. The darker the value, the more similar the point is to the clicked image point.

The result should look similar to this:

_images/narf_descriptor_visualization.png

Also have a look at:

$ ./narf_descriptor_visualization -h

for a list of parameters.