diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 091cb0640..338b908c9 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -20,6 +20,7 @@ endif () if (OpenCV_FOUND) add_subdirectory(green_screen) add_subdirectory(opencv_compatibility) + add_subdirectory(depth_eval_tools) if ("${CMAKE_SYSTEM_NAME}" STREQUAL "Windows") file(COPY ${OpenCV_DIR}/../bin/ DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} FILES_MATCHING PATTERN "*.dll") diff --git a/examples/depth_eval_tools/CMakeLists.txt b/examples/depth_eval_tools/CMakeLists.txt new file mode 100644 index 000000000..47146da89 --- /dev/null +++ b/examples/depth_eval_tools/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +add_subdirectory(collect) +add_subdirectory(depth_eval) +add_subdirectory(mkv2images) +add_subdirectory(transformation_eval) + diff --git a/examples/depth_eval_tools/README.md b/examples/depth_eval_tools/README.md new file mode 100644 index 000000000..cf2cc98e0 --- /dev/null +++ b/examples/depth_eval_tools/README.md @@ -0,0 +1,226 @@ +# Azure Kinect - Depth Evaluation Tools Examples + +## Description + + Depth Evaluation Example Tools for Azure Kinect DK. + + These tools enable developers to collect depth and color images from a Azure Kinect device (`collect`), convert mkv files to images (`mkv2images`), evaluate the depth bias (`depth_eval`), and evaluate the transformation re-projection error between sensors (`transformation_eval`). + +--- +## Setup + + * Print out one of the three target files targets provided in the repo: `plane_files\plane.pdf`, `plane_files\plane_large.pdf` or `plane_files\plane_small.pdf`. + + * The small plane file is sized to fit on an 8.5"x11" piece of paper. The large plane file is twice the size of `plane.pdf`. + + * `plane.json`, `plane_large.json` and `plane_small.json` define the physical parameters of the target board. Square length is the length of one side in mm of the charuco_square, marker_length is the size length of the QR code marker in mm. You can use opencv to create your own Charuco target. In this case you would need to copy and modify the json parameters accordingly. + + * Parameter aruco_dict_name is an ENUM specifying the tag type. All plane files use dictionary number six. + + * See predefined dictionaries on the OpenCV website [here.](https://docs.opencv.org/master/dc/df7/dictionary_8hpp.html) + ![Board Params](example_images/plane_parameters/board_parameters.png "Board Parameters") + + * To capture good data, it is recommended to capture images of the target board with the camera(s) aligned with the center of the target and from a reasonable distance such that the target board fills the majority of the view. The target should be as flat as possible. It is also best to avoid being low to the floor and to minimize reflections in the space. See the provided example data for reference. + + * For high quality data, the image should _not_ be grainy and you should be able to visually see all fiducial features on the board. + + * The majority of the tools take a MKV file as an input. A good way to capture data is to use the Azure Kinect DK recorder. Example command: ```k4arecorder.exe -c 3072p -d PASSIVE_IR -l 5 board1.mkv``` + + * Good example: The fiducial lines are clear and distinct even to the human eye. + + * To verify the collected data is of high quality, use the `mkv2images` tool to view the data collected. + + ![High Quality IR Image](example_images/collect/ir8-0.png "High Quality IR Image") + + ![High Quality Color Image](example_images/collect/color-0.png "High Quality Color Image") + +--- +## Using the `collect` Tool to Capture Images + + * This tool is used to collect color, passive IR, and/or depth images from the device. + + * Based on the number of view specified (-nv), this tool will pause after each capture to allow the user to move the device to the next position. + + * This tool is a good aid for collecting images for use with the Calibration and Registration tools. + + * Minimum example command: ```./collect -mode=3 -res=1 -nv=2 -nc=10 -cal=1 -out=c:/data``` + + * The following are all the options exposed by this tool, see tool specific [README](collect/README.md) for more information: + + ``` + ./collect -h or -help or -? print the help message + ./collect -mode= -res= -nv= -nc= + -fps= -cal= -xy= -d= -i= -c= + -out= + -gg= + -gm= + -gp= + -av=<0:dump mean images only, 1:dump all images, 2:dump all images and their mean> + ``` + +--- +## Using the `mkv2images` Tool to Convert MKV captures to Images + + * This tool is used to dump png images from a provided MKV file. + + * This tool should be used to verify that the collected mkv data is of high enough quality as mentioned in the setup section. + + * It is recommended that MKV files are collected using the Azure Kinect DK recorder. + + * Example command: ```k4arecorder.exe -c 3072p -d PASSIVE_IR -l 5 board1.mkv``` + + * Minimum example command: ```./mkv2images -in=board1.mkv -out=c:/data -c=0 -f=0``` + + * The following are all the options exposed by this tool, see tool specific [README](mkv2images/README.md) for more information: + + ``` + ./mkv2images -h or -help or -? print the help message + ./mkv2images -in= -out= -d= -i= -c= + -f=<0:dump mean images only, 1 : dump first frame> + -gg= + -gm= + -gp= + ``` + +--- +## Using the `depth_eval` Tool to Evaluate Depth Bias + + * This tool is used to evaluate the depth bias of a device. + + * This tool requires two MKV files as input, one captured using PASSIVE_IR and the other using WFOV_2X2BINNED. These two files should be collected with the camera and target board setup unchanged. + + * The Passive IR MKV file should be collected using the following command: ```k4arecorder.exe -c 3072p -d PASSIVE_IR -l 3 board1.mkv``` + + * The Depth MKV file should be collected using the following command: ```k4arecorder.exe -c 3072p -d WFOV_2X2BINNED -l 3 board2.mkv``` + + * This tool will evaluate the depth bias of the device and output the results to the console. + + * The output consists of four values. Total charuco corners as specified by the charuco dictionary, the actual number of detected corners (Depends on image quality, the higher the better), the Mean Z depth bias in millimeters, and the RMS Z depth bias in millimeters. + + * Depth bias is the difference between the ground truth depth measurement (determined by the projection of the target board) and the measured depth from the sensor. + + * Example Output: + + ``` + board has 104 charuco corners + number of detected corners in ir = 101 + Mean of Z depth bias = 3.33104 mm + RMS of Z depth bias = 3.47157 mm + ``` + + * The maximum bias should be expected to be within +/- 11mm. A depth bias outside of this range indicates a poor device calibration and the example Calibration tool should be used to obtain an improved calibration that can be stored (external of the device) and used in place of the factory calibration (Note: The factory calibration stored on the Kinect device can not be overwritten). + + * Minimum example command: ```./depth_eval -i=board1.mkv -d=board2.mkv -t=plane.json -out=c:/data``` + + * The following are all the options exposed by this tool, see tool specific [README](depth_eval/README.md) for more information: + + ``` + ./depth_eval -h or -help or -? print the help message + ./depth_eval -i= -d= -t= + -out= -s=<1:generate and save result images> + -gg= + -gm= + -gp= + ``` + +--- +## Using the `transformation_eval` Tool to Evaluate Transformation Mapping Between Sensors + + * This tool is used to evaluate the transformation between the sensors of a single device. + + * This tool requires two MKV files as input, one captured using PASSIVE_IR and the other using WFOV_2X2BINNED. These two files should be collected with the camera and target board setup unchanged. + + * The Passive IR MKV file should be collected using the following command: ```k4arecorder.exe -c 3072p -d PASSIVE_IR -l 3 board1.mkv``` + + * The Depth MKV file should be collected using the following command: ```k4arecorder.exe -c 3072p -d WFOV_2X2BINNED -l 3 board2.mkv``` + + * This tool will evaluate the transformation re-projection error between the color camera and depth sensor of a device. + + * The output consists of five values. Total charuco corners as specified by the charuco dictionary, the actual number of detected corners in IR capture (_Depends on image quality, the higher the better_), the actual number of detected corners in the color capture (_Depends on image quality, the higher the better_), the number of common corners detected between the images, and the RMS re-projection error in pixels. + + * Use the `-s` flag to generate and save visual representations of the transformation re-projection error. + + * The output image, `checkered_pattern.png`, is a visual representation of the sensor registration error. The image is a composition of the color and IR images. Zooming in, the registration error (alignment of Charuco target pattern) can be clearly seen at the boundaries of the checkerboard pattern. + + * The output image, `transformation_error.png`, is a visual of the re-projection error represented as a vector field. Zooming in, the green mark is the location of the detected marker corner in the color image, and the blue marker represents the projected position of the detected marker corner in the IR image. + + * Re-projection error is the difference between the position of the target in the color image and the target as captured in the IR image projected into the coordinate space of the color camera. + + ![Reprojection Error Calculation](example_images/transformation_eval/tr_eval.png "Reprojection Error Calculation") + + * Example Output: + + ``` + board has 104 charuco corners + corners detected in ir = 73 + corners detected in color = 93 + number of common corners = 65 + rms = 7.42723 pixels + ``` + + * The RMS re-projection error should be no greater than 12 pixels. An RMS re-projection error greater than this indicates a poor device registration and the example Registration tool should be used to correct the device registration. + + * Minimum example command: ```./transformation_eval -i=board1.mkv -d=board2.mkv -t=plane.json -out=c:/data``` + + * The following are all the options exposed by this tool, see tool specific [README](transformation_eval/README.md) for more information: + + ``` + ./transformation_eval -h or -help or -? print the help message + ./transformation_eval -i= -d= -t= + -out= -s=<1:generate and save result images> + -gg= + -gm= + -gp= + ``` +--- +## OpenCV Dependency + + These example tools require both OpenCV and OpenCV Contrib to be installed prior to building the SDK. After installing OpenCV Contrib, the lib path in ```cmake/FindOpenCV.cmake``` will need to change from ```C:/opencv/build/x64/vc14/lib``` to ```C:/opencv/build/x64/vc16/lib``` + + If a prior version of OpenCV exists in ```C:\opencv```, remove it before running the following steps. + + To build opencv and opencv_contrib from source follow these steps: + + [General Instalation Toutorial](https://docs.opencv.org/4.5.0/d0/d3d/tutorial_general_install.html) + + [OpenCV configuration options](https://docs.opencv.org/master/db/d05/tutorial_config_reference.html) + + 1. Start an instance of "x64 Native Tools Command Prompt for VS 2019" + + 2. Clone opencv and opencv_contrib: + + ```c:\> git clone https://github.com/opencv/opencv && git -C opencv checkout 4.5.0``` + + ```c:\> git clone https://github.com/opencv/opencv_contrib && git -C opencv_contrib checkout 4.5.0``` + + 3. Build Release Version + + ```c:\> cd opencv && mkdir build && cd build``` + + ```c:\opencv\build> cmake .. -GNinja -DOPENCV_EXTRA_MODULES_PATH=../../opencv_contrib/modules -DBUILD_opencv_world=ON -DCMAKE_BUILD_TYPE=Release -DBUILD_PERF_TESTS:BOOL=OFF -DBUILD_TESTS:BOOL=OFF -DCMAKE_INSTALL_PREFIX=c:/opencv/build``` + + 4. Install Release Version + + ```c:\opencv\build> cd ..``` + + ```c:\opencv> cmake --build c:/opencv/build --target install``` + + 5. Build Debug Version + + ```c:>mkdir build_debug && cd build_debug``` + + ```c:\opencv\build_debug>cmake .. -GNinja -DOPENCV_EXTRA_MODULES_PATH=../../opencv_contrib/modules -DBUILD_opencv_world=ON -DCMAKE_BUILD_TYPE=Debug -DBUILD_PERF_TESTS:BOOL=OFF -DBUILD_TESTS:BOOL=OFF -DCMAKE_INSTALL_PREFIX=c:/opencv/build``` + + 6. Install Debug Version + + ```c:\opencv\build_debug> cd ..``` + + ```c:\opencv> cmake --build c:/opencv/build_debug --target install``` + + + ***NOTE*** + + * The default install location for opencv is `c:\opencv\build\install\...` + * However the Azure-Kinect-Sensor-SDK expects an install at `c:\opencv\build\...` + * To change the default install location add `-DCMAKE_INSTALL_PREFIX=` + to the `cmake .. -GNinja` command diff --git a/examples/depth_eval_tools/collect/CMakeLists.txt b/examples/depth_eval_tools/collect/CMakeLists.txt new file mode 100644 index 000000000..d388b0584 --- /dev/null +++ b/examples/depth_eval_tools/collect/CMakeLists.txt @@ -0,0 +1,23 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +find_package(OpenCV REQUIRED) +include_directories( + . + ./inc/ + ../kahelpers/ + ../kahelpers/inc/ + ) + +add_executable(collect + collect.cpp + ./inc/collect.h + ../kahelpers/kahelpers.cpp + ../kahelpers/inc/kahelpers.h + ) + +target_link_libraries(collect PRIVATE + k4a::k4a + k4a::k4arecord + ${OpenCV_LIBS} + ) \ No newline at end of file diff --git a/examples/depth_eval_tools/collect/README.md b/examples/depth_eval_tools/collect/README.md new file mode 100644 index 000000000..629c58fb5 --- /dev/null +++ b/examples/depth_eval_tools/collect/README.md @@ -0,0 +1,69 @@ +# Azure Kinect - Depth Evaluation Tools Examples - collect + +--- + +## Description + + Collect multiple view, depth and color images from K4A device and pre-process the data for use with the other evaluation tools. + +--- + +## Usage + ``` + ./collect -h or -help or -? print the help message + ./collect -mode= -res= -nv= -nc= + -fps= -cal= -xy= -d= -i= -c= + -out= + -gg= + -gm= + -gp= + -av=<0:dump mean images only, 1:dump all images, 2:dump all images and their mean> + ``` + + Example Command: ```./collect -mode=3 -res=1 -nv=2 -nc=10 -cal=1 -out=c:/data``` + + --- + --- + + Depth mode can be [0, 1, 2, 3, 4 or 5] as follows: + + | Depth Mode | Details | + | :--------- | :------ | + | K4A_DEPTH_MODE_OFF = 0 | 0:Depth sensor will be turned off with this setting. | + | K4A_DEPTH_MODE_NFOV_2X2BINNED | 1:Depth captured at 320x288. Passive IR is also captured at 320x288. | + | K4A_DEPTH_MODE_NFOV_UNBINNED | 2:Depth captured at 640x576. Passive IR is also captured at 640x576. | + | K4A_DEPTH_MODE_WFOV_2X2BINNED | 3:Depth captured at 512x512. Passive IR is also captured at 512x512. | + | K4A_DEPTH_MODE_WFOV_UNBINNED | 4:Depth captured at 1024x1024. Passive IR is also captured at 1024x1024. | + | K4A_DEPTH_MODE_PASSIVE_IR | 5:Passive IR only, captured at 1024x1024. | + + --- + --- + + Color resolution can be [0, 1, 2, 3, 4, 5, or 6] as follows: + + | Color Resolution | Details | + | :--------------- | :------ | + | K4A_COLOR_RESOLUTION_OFF = 0 | 0: Color camera will be turned off. | + | K4A_COLOR_RESOLUTION_720P | 1: 1280 * 720 16:9. | + | K4A_COLOR_RESOLUTION_1080P | 2: 1920 * 1080 16:9. | + | K4A_COLOR_RESOLUTION_1440P | 3: 2560 * 1440 16:9. | + | K4A_COLOR_RESOLUTION_1536P | 4: 2048 * 1536 4:3. | + | K4A_COLOR_RESOLUTION_2160P | 5: 3840 * 2160 16:9. | + | K4A_COLOR_RESOLUTION_3072P | 6: 4096 * 3072 4:3. | + + --- + --- + + FPS can be [0, 1, or 2] as follows: + + | FPS Mode | Details | + | :------- | :------ | + | K4A_FRAMES_PER_SECOND_5 = 0 | 0: FPS=5. | + | K4A_FRAMES_PER_SECOND_15 | 1: FPS=15. | + | K4A_FRAMES_PER_SECOND_30 | 2: FPS=30. | + + --- + +## Dependencies + + OpenCV diff --git a/examples/depth_eval_tools/collect/collect.cpp b/examples/depth_eval_tools/collect/collect.cpp new file mode 100644 index 000000000..dae1324d4 --- /dev/null +++ b/examples/depth_eval_tools/collect/collect.cpp @@ -0,0 +1,296 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include +#include +#include +#include + +#include // std::cout +#include // std::ofstream + +#include +#include +#include +#include + +#include "inc/collect.h" +#include "kahelpers.h" + +using namespace kahelpers; + +void help() +{ + std::cout + << "\nCollect depth and color images from K4A.\n" + "Usage:\n" + "./collect -h or -help or -? print this help message\n" + "./collect -mode= -res= -nv= -nc= " + " -fps= -cal= -xy= -d= -i= " + "-c=" + " -out=\n" + " -gg=\n" + " -gm=\n" + " -gp=\n" + " -av=<0:dump mean images only, 1:dump all images, 2:dump all images and their mean>\n" + " Example:\n" + "./collect -mode=3 -res=1 -nv=2 -nc=10 -cal=1 -out=c:/data\n"; + std::cout << "\n---\ndepth mode can be [0, 1, 2, 3, 4 or 5] as follows\n"; + std::cout << "K4A_DEPTH_MODE_OFF = 0, 0:Depth sensor will be turned off with this setting.\n"; + std::cout + << "K4A_DEPTH_MODE_NFOV_2X2BINNED, 1:Depth captured at 320x288. Passive IR is also captured at 320x288.\n"; + std::cout + << "K4A_DEPTH_MODE_NFOV_UNBINNED, 2:Depth captured at 640x576. Passive IR is also captured at 640x576.\n"; + std::cout + << "K4A_DEPTH_MODE_WFOV_2X2BINNED, 3:Depth captured at 512x512. Passive IR is also captured at 512x512.\n"; + std::cout + << "K4A_DEPTH_MODE_WFOV_UNBINNED, 4:Depth captured at 1024x1024. Passive IR is also captured at 1024x1024.\n"; + std::cout << "K4A_DEPTH_MODE_PASSIVE_IR, 5:Passive IR only, captured at 1024x1024. \n"; + std::cout << "\n---\ncolor resolution can be [0, 1, 2, 3, 4, 5, or 6] as follows\n"; + std::cout << "K4A_COLOR_RESOLUTION_OFF = 0, 0: Color camera will be turned off.\n"; + std::cout << "K4A_COLOR_RESOLUTION_720P, 1: 1280 * 720 16:9. \n"; + std::cout << "K4A_COLOR_RESOLUTION_1080P, 2: 1920 * 1080 16:9. \n"; + std::cout << "K4A_COLOR_RESOLUTION_1440P, 3: 2560 * 1440 16:9. \n"; + std::cout << "K4A_COLOR_RESOLUTION_1536P, 4: 2048 * 1536 4:3. \n"; + std::cout << "K4A_COLOR_RESOLUTION_2160P, 5: 3840 * 2160 16:9. \n"; + std::cout << "K4A_COLOR_RESOLUTION_3072P, 6: 4096 * 3072 4:3. \n"; + std::cout << "\n---\nfps can be [0, 1, or 2] as follows\n"; + std::cout << "K4A_FRAMES_PER_SECOND_5 = 0, 0: FPS=5. \n"; + std::cout << "K4A_FRAMES_PER_SECOND_15, 1: FPS=15. \n"; + std::cout << "K4A_FRAMES_PER_SECOND_30, 2: FPS=30. \n"; +} +int main(int argc, char **argv) +{ + + cv::CommandLineParser + parser(argc, + argv, + "{help h usage ?| |print this message}" + "{mode| | depth mode:0, 1, 2, 3, 4 or 5}" + "{res| | color res:0, 1, 2, 3, 4, 5 or 6}" + "{out| | output dir}" + "{nv|1| number of views}" + "{nc|1| number of captures per view}" + "{fps|0| frame rate per sec}" + "{cal|0| dump calibration}" + "{xy|0| dump xy calibration table}" + "{d|1| capture depth}" + "{i|1| capture ir}" + "{c|1| capture color}" + "{gg|0.5| gray_gamma used to convert ir data to 8bit gray image}" + "{gm|4000.0| gray_max used to convert ir data to 8bit gray image}" + "{gp | 99.0 | percentile used to convert ir data to 8bit gray image}" + "{av|0| 0:dump mean images only, 1:dump all images, 2:dump all images and their mean}"); + // get input argument + + if (parser.has("help") || !parser.has("mode") || !parser.has("res") || !parser.has("out")) + { + help(); + return 0; + } + + int mode = parser.get("mode"); + int res = parser.get("res"); + std::string output_dir = parser.get("out"); + + int num_views = parser.get("nv"); + int num_caps_per_view = parser.get("nc"); + + int fps = parser.get("fps"); + + bool dump_calibration = parser.get("cal") > 0; + bool dump_xytable = parser.get("xy") > 0; + + bool dump_depth = parser.get("d") > 0; + bool dump_ir = parser.get("i") > 0; + bool dump_color = parser.get("c") > 0; + + int av = parser.get("av"); + + float gray_gamma = parser.get("gg"); + float gray_max = parser.get("gm"); + float gray_percentile = parser.get("gp"); + + uint32_t device_count = k4a::device::get_installed_count(); + + if (device_count == 0) + { + std::cerr << "No K4A devices found\n"; + return 1; + } + + k4a::device device = k4a::device::open(K4A_DEVICE_DEFAULT); + + k4a_device_configuration_t config = K4A_DEVICE_CONFIG_INIT_DISABLE_ALL; + config.depth_mode = (k4a_depth_mode_t)mode; + config.color_format = K4A_IMAGE_FORMAT_COLOR_MJPG; + config.color_resolution = (k4a_color_resolution_t)res; + config.camera_fps = (k4a_fps_t)fps; + + device.start_cameras(&config); + + k4a::calibration calibration = device.get_calibration(config.depth_mode, config.color_resolution); + + if (dump_calibration) + { + write_calibration_blob(device.get_raw_calibration(), output_dir, "calibration_blob"); + write_opencv_calib(calibration.depth_camera_calibration, output_dir, "cal_depth_mode" + std::to_string(mode)); + write_opencv_calib(calibration.color_camera_calibration, output_dir, "cal_color"); + } + + if (dump_xytable) + { + k4a::image xy_table = k4a::image::create(K4A_IMAGE_FORMAT_CUSTOM, + calibration.depth_camera_calibration.resolution_width, + calibration.depth_camera_calibration.resolution_height, + calibration.depth_camera_calibration.resolution_width * + (int)sizeof(k4a_float2_t)); + + create_xy_table(calibration, xy_table); + write_xy_table(xy_table, output_dir, "xy_table_mode" + std::to_string(mode)); + } + + k4a::capture capture; + + std::chrono::milliseconds timeout_msec(1000); + + std::cout << "Capturing " << num_caps_per_view << " frames per view \n"; + for (int view_idx = 0; view_idx < num_views; view_idx++) + { + std::cout << "Ready to capture next frame . Press Enter? \n"; + // system("pause"); + getchar(); + cv::Mat mean_ir, mean_depth, mean_color[3]; + int n_ir = 0, n_depth = 0, n_color = 0; + + for (int cap_idx = 0; cap_idx < num_caps_per_view; cap_idx++) + { + // Get a capture + if (!device.get_capture(&capture, timeout_msec)) + { + std::cerr << "Timed out waiting for a capture\n"; + break; + } + k4a::image depth_img; + k4a::image ir_img; + k4a::image color_img; + + if (dump_color) + { + // get color image + color_img = capture.get_color_image(); + if (color_img.is_valid()) + { + std::cout << " | Color res: " << color_img.get_height_pixels() << "x" + << color_img.get_width_pixels(); + cv::Mat frame32[3]; + cv::Mat bands[3]; + cv::Mat M = color_to_opencv(color_img); + cv::split(M, bands); + for (int ch = 0; ch < 3; ch++) + { + bands[ch].convertTo(frame32[ch], CV_32F); + if (n_color < 1) + mean_color[ch] = frame32[ch].clone(); + else + mean_color[ch] += frame32[ch].clone(); + } + + if (av > 0) + { + std::string tstr = std::to_string(color_img.get_device_timestamp().count()); + cv::imwrite(output_dir + "/" + tstr + "-color.png", M); + } + n_color++; + } + } + + if (dump_depth) + { + // get depth image + depth_img = capture.get_depth_image(); + if (depth_img.is_valid()) + { + std::cout << " | Depth16 res: " << depth_img.get_height_pixels() << "x" + << depth_img.get_width_pixels(); + + cv::Mat frame32; + cv::Mat M = depth_to_opencv(depth_img); + M.convertTo(frame32, CV_32F); + if (n_depth < 1) + mean_depth = frame32.clone(); + else + mean_depth += frame32.clone(); + + if (av > 0) + { + std::string tstr = std::to_string(depth_img.get_device_timestamp().count()); + cv::imwrite(output_dir + "/" + tstr + "-depth16.png", M); + } + n_depth++; + } + } + + if (dump_ir) + { + // get ir image + ir_img = capture.get_ir_image(); + if (ir_img.is_valid()) + { + std::cout << " | Ir16 res: " << ir_img.get_height_pixels() << "x" << ir_img.get_width_pixels(); + cv::Mat frame32; + cv::Mat M = ir_to_opencv(ir_img); + M.convertTo(frame32, CV_32F); + if (n_ir < 1) + mean_ir = frame32.clone(); + else + mean_ir += frame32.clone(); + + if (av > 0) + { + std::string tstr = std::to_string(ir_img.get_device_timestamp().count()); + cv::imwrite(output_dir + "/" + tstr + "-ir16.png", M); + } + n_ir++; + } + } + std::cout << std::endl; + capture.reset(); + } + + if (n_ir > 0 && av != 1) + { + mean_ir /= n_ir; + cv::Mat ir8, ir16; + get_gray_gamma_img(mean_ir, ir8, gray_gamma, gray_max, gray_percentile); + mean_ir.convertTo(ir16, CV_16U); + cv::imwrite(output_dir + "\\" + "ir8-" + std::to_string(view_idx) + ".png", ir8); + cv::imwrite(output_dir + "\\" + "ir16-" + std::to_string(view_idx) + ".png", ir16); + } + if (n_depth > 0 && av != 1) + { + mean_depth /= n_depth; + cv::Mat depth16; + mean_depth.convertTo(depth16, CV_16U); + cv::imwrite(output_dir + "\\" + "depth16-" + std::to_string(view_idx) + ".png", depth16); + } + if (n_color > 1 && av != 1) + { + cv::Mat channels[3]; + for (int ch = 0; ch < 3; ch++) + { + mean_color[ch] /= n_color; + mean_color[ch].convertTo(channels[ch], CV_8U); + } + cv::Mat color8; + cv::merge(channels, 3, color8); + cv::imwrite(output_dir + "\\" + "color-" + std::to_string(view_idx) + ".png", color8); + } + } + + device.stop_cameras(); + device.close(); + + return 0; +} \ No newline at end of file diff --git a/examples/depth_eval_tools/collect/inc/collect.h b/examples/depth_eval_tools/collect/inc/collect.h new file mode 100644 index 000000000..e120d164c --- /dev/null +++ b/examples/depth_eval_tools/collect/inc/collect.h @@ -0,0 +1,8 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#ifndef COLLECT_H +#define COLLECT_H + +void help(); + +#endif diff --git a/examples/depth_eval_tools/depth_eval/CMakeLists.txt b/examples/depth_eval_tools/depth_eval/CMakeLists.txt new file mode 100644 index 000000000..8dacb0779 --- /dev/null +++ b/examples/depth_eval_tools/depth_eval/CMakeLists.txt @@ -0,0 +1,23 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +find_package(OpenCV REQUIRED) +include_directories( + . + ./inc/ + ../kahelpers/ + ../kahelpers/inc/ + ) + +add_executable(depth_eval + depth_eval.cpp + ./inc/depth_eval.h + ../kahelpers/kahelpers.cpp + ../kahelpers/inc/kahelpers.h + ) + +target_link_libraries(depth_eval PRIVATE + k4a::k4a + k4a::k4arecord + ${OpenCV_LIBS} + ) \ No newline at end of file diff --git a/examples/depth_eval_tools/depth_eval/README.md b/examples/depth_eval_tools/depth_eval/README.md new file mode 100644 index 000000000..d74619efc --- /dev/null +++ b/examples/depth_eval_tools/depth_eval/README.md @@ -0,0 +1,36 @@ +# Azure Kinect - Depth Evaluation Tools Examples - depth_eval + +--- + +## Description + + Depth Evaluation Tool for K4A. + + This tool utilizes two mkv files. + + The 1st mkv file is PASSIVE_IR recorded using: ```k4arecorder.exe -c 3072p -d PASSIVE_IR -l 3 board1.mkv``` + + The 2nd mkv file is WFOV_2X2BINNED recorded using: ```k4arecorder.exe -c 3072p -d WFOV_2X2BINNED -l 3 board2.mkv``` + + This version supports WFOV_2X2BINNED but can be easily generalized. + +--- + +## Usage + + ``` + ./depth_eval -h or -help or -? print the help message + ./depth_eval -i= -d= -t= + -out= -s=<1:generate and save result images> + -gg= + -gm= + -gp= + ``` + + Example Command: ```./depth_eval -i=board1.mkv -d=board2.mkv -t=plane.json -out=c:/data``` + +--- +## Dependencies + + OpenCV + OpenCV Contrib diff --git a/examples/depth_eval_tools/depth_eval/depth_eval.cpp b/examples/depth_eval_tools/depth_eval/depth_eval.cpp new file mode 100644 index 000000000..06ad29a49 --- /dev/null +++ b/examples/depth_eval_tools/depth_eval/depth_eval.cpp @@ -0,0 +1,246 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include // std::cout +#include // std::ofstream + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "inc/depth_eval.h" +#include "kahelpers.h" + +using namespace kahelpers; + +void help() +{ + std::cout << "\nDepth Evaluation Tool for K4A.\n"; + std::cout << "\nit uses 2 mkv files:\n"; + std::cout + << "\t 1st is PASSIVE_IR recorded using: \n\t\t k4arecorder.exe -c 3072p -d PASSIVE_IR -l 3 board1.mkv\n"; + std::cout << "\t 2nd is WFOV_2X2BINNED recorded using: \n\t\t k4arecorder.exe -c 3072p -d WFOV_2X2BINNED -l 3 " + "board2.mkv\n"; + std::cout << "\t This version supports WFOV_2X2BINNED but can be easily generalized\n"; + std::cout << "Usage:\n" + "./depth_eval -h or -help or -? print this help message\n" + "./depth_eval -i= -d= -t=" + " -out= -s=<1:generate and save result images>\n" + " -gg=\n" + " -gm=\n" + " -gp=\n" + " Example:\n" + "./depth_eval -i=board1.mkv -d=board2.mkv -t=plane.json -out=c:/data\n"; +} + +// +// Timestamp in milliseconds. Defaults to 1 sec as the first couple frames don't contain color +static bool process_mkv(const std::string &passive_ir_mkv, + const std::string &depth_mkv, + const std::string &template_file, + int timestamp, + const std::string &output_dir, + float gray_gamma, + float gray_max, + float gray_percentile, + bool save_images) +{ + // + // get passive ir + k4a::playback playback_ir = k4a::playback::open(passive_ir_mkv.c_str()); + if (playback_ir.get_calibration().depth_mode != K4A_DEPTH_MODE_PASSIVE_IR) + { + std::cerr << "depth_mode != K4A_DEPTH_MODE_PASSIVE_IR"; + return false; + } + cv::Mat passive_ir; + cv::Mat nullMat; + get_images(playback_ir, timestamp, passive_ir, nullMat, nullMat, false, true, false, false); + // generate a 8bit gray scale image from the passive ir so it can be used for marker detection + cv::Mat passive_ir8; + get_gray_gamma_img(passive_ir, passive_ir8, gray_gamma, gray_max, gray_percentile); + + // + // get depth + k4a::playback playback = k4a::playback::open(depth_mkv.c_str()); + // K4A_DEPTH_MODE_WFOV_2X2BINNED : Depth captured at 512x512 + // this version of the code supports WFOV_2X2BINNED but can be easily generalized, see switch statement below + k4a_depth_mode_t depth_mode = playback.get_calibration().depth_mode; + if (depth_mode != K4A_DEPTH_MODE_WFOV_2X2BINNED) + { + std::cerr << "depth_mode != K4A_DEPTH_MODE_WFOV_2X2BINNED"; + return false; + } + + cv::Mat ir16, depth16; + get_images(playback, timestamp, ir16, depth16, nullMat, false, true, true, false); + + cv::Mat ir8; + get_gray_gamma_img(ir16, ir8, gray_gamma, gray_max, gray_percentile); + + if (save_images) + { + // save images as png files + if (!ir16.empty()) + cv::imwrite(output_dir + "/ir16.png", ir16); + if (!depth16.empty()) + cv::imwrite(output_dir + "/depth16.png", depth16); + if (!passive_ir8.empty()) + cv::imwrite(output_dir + "/color8.png", passive_ir8); + if (!ir8.empty()) + cv::imwrite(output_dir + "/ir8.png", ir8); + } + + // + // create a charuco target from a json template + charuco_target charuco(template_file); + cv::Ptr board = charuco.create_board(); + // + // detect markers in passive_ir8 + cv::Ptr params = cv::aruco::DetectorParameters::create(); + // Parameter not in OpenCV 3.2.0 (used by build pipeline) + // params->cornerRefinementMethod = cv::aruco::CORNER_REFINE_NONE; // best option as my limited testing indicated + std::vector markerIds_ir; + std::vector> markerCorners_ir; + std::vector charucoIds_ir; + std::vector charucoCorners_ir; + + detect_charuco(passive_ir8, board, params, markerIds_ir, markerCorners_ir, charucoIds_ir, charucoCorners_ir, false); + + std::cout << "\n board has " << board->chessboardCorners.size() << " charuco corners"; + std::cout << "\n number of detected corners in ir = " << charucoIds_ir.size(); + + // + // get camera calibration + k4a::calibration calibration = playback_ir.get_calibration(); + cv::Mat camera_matrix; + cv::Mat dist_coeffs; + cv::Mat rvec, tvec; + calibration_to_opencv(calibration.depth_camera_calibration, camera_matrix, dist_coeffs); + // + // estimate pose of the board + bool converged = cv::aruco::estimatePoseCharucoBoard( + charucoCorners_ir, charucoIds_ir, board, camera_matrix, dist_coeffs, rvec, tvec); + + // + // use camera intrinsics and pose to generate ground truth points + std::vector corners3d; + std::vector corners3d_cam; // 3d corners in camera coord. = ground truth + get_board_object_points_charuco(board, charucoIds_ir, corners3d); + + if (converged) + { + cv::Mat R; + cv::Rodrigues(rvec, R); + for (unsigned int i = 0; i < corners3d.size(); i++) + { + cv::Mat Pw = (cv::Mat_(3, 1) << corners3d[i].x, corners3d[i].y, corners3d[i].z); + + cv::Mat Pcam = R * Pw + tvec; + corners3d_cam.push_back( + cv::Point3f((float)Pcam.at(0, 0), (float)Pcam.at(1, 0), (float)Pcam.at(2, 0))); + } + } + + std::vector dz; + + for (unsigned int i = 0; i < corners3d_cam.size(); i++) + { + float d_mm; + cv::Point2f corner_depth; + // modify this to support other depth modes + switch (depth_mode) + { + case K4A_DEPTH_MODE_WFOV_2X2BINNED: + // for passive ir res (1024) to depth res (512) + corner_depth.x = charucoCorners_ir[i].x / 2; + corner_depth.y = charucoCorners_ir[i].y / 2; + break; + default: + corner_depth.x = charucoCorners_ir[i].x / 2; + corner_depth.y = charucoCorners_ir[i].y / 2; + } + + if (interpolate_depth(depth16, corner_depth.x, corner_depth.y, d_mm)) + { + // bias = measured depth - ground truth + if (d_mm > 0) + dz.push_back(d_mm - 1000.0f * corners3d_cam[i].z); + } + } + + float dz_mean = 0.0; + float dz_rms = 0.0; + for (unsigned int i = 0; i < dz.size(); i++) + { + // std::cout << std::endl << dz[i]; + dz_mean += dz[i]; + dz_rms += dz[i] * dz[i]; + } + dz_mean /= dz.size(); + dz_rms = (float)sqrt(dz_rms / dz.size()); + std::cout << std::endl << "Mean of Z deph bias = " << dz_mean << " mm"; + std::cout << std::endl << "RMS of Z depth bias = " << dz_rms << " mm"; + + return true; +} + +int main(int argc, char **argv) +{ + + cv::CommandLineParser parser(argc, + argv, + "{help h usage ?| |print this message}" + "{i| | full path of the passive_ir mkv file}" + "{d| | full path of the wfov_binned mkv file}" + "{t| | full path of the board json file e.g., /plane.json}" + "{out| | full path of the output dir}" + "{s|1| generate and save result images}" + "{gg|0.5| gray_gamma used to convert ir data to 8bit gray image}" + "{gm|4000.0| gray_max used to convert ir data to 8bit gray image}" + "{gp|99.0| percentile used to convert ir data to 8bit gray image}"); + // get input argument + std::string passive_ir_mkv = parser.get("i"); + std::string depth_mkv = parser.get("d"); + std::string template_file = parser.get("t"); + std::string output_dir = parser.get("out"); + + if (passive_ir_mkv.empty() || depth_mkv.empty() || template_file.empty() || output_dir.empty()) + { + help(); + return 1; + } + bool save_images = parser.get("s") > 0; + + float gray_gamma = parser.get("gg"); + float gray_max = parser.get("gm"); + float gray_percentile = parser.get("gp"); + + if (parser.has("help")) + { + help(); + return 0; + } + + // Timestamp in milliseconds. Defaults to 1 sec as the first couple frames don't contain color + int timestamp = 1000; + + bool stat = process_mkv(passive_ir_mkv, + depth_mkv, + template_file, + timestamp, + output_dir, + gray_gamma, + gray_max, + gray_percentile, + save_images); + + return (stat) ? 0 : 1; +} diff --git a/examples/depth_eval_tools/depth_eval/inc/depth_eval.h b/examples/depth_eval_tools/depth_eval/inc/depth_eval.h new file mode 100644 index 000000000..adf3f9940 --- /dev/null +++ b/examples/depth_eval_tools/depth_eval/inc/depth_eval.h @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#ifndef DEPTH_EVAL_H +#define DEPTH_EVAL_H + +#include + +void help(); + +static bool process_mkv(const std::string &passive_ir_mkv, + const std::string &depth_mkv, + const std::string &template_file, + int timestamp, + const std::string &output_dir, + float gray_gamma, + float gray_max, + float gray_percentile, + bool save_images); + +#endif diff --git a/examples/depth_eval_tools/example_images/collect/color-0.png b/examples/depth_eval_tools/example_images/collect/color-0.png new file mode 100644 index 000000000..993e608f8 Binary files /dev/null and b/examples/depth_eval_tools/example_images/collect/color-0.png differ diff --git a/examples/depth_eval_tools/example_images/collect/ir8-0.png b/examples/depth_eval_tools/example_images/collect/ir8-0.png new file mode 100644 index 000000000..d4d23253b Binary files /dev/null and b/examples/depth_eval_tools/example_images/collect/ir8-0.png differ diff --git a/examples/depth_eval_tools/example_images/depth_eval/color8.png b/examples/depth_eval_tools/example_images/depth_eval/color8.png new file mode 100644 index 000000000..cc03e1803 Binary files /dev/null and b/examples/depth_eval_tools/example_images/depth_eval/color8.png differ diff --git a/examples/depth_eval_tools/example_images/depth_eval/depth16.png b/examples/depth_eval_tools/example_images/depth_eval/depth16.png new file mode 100644 index 000000000..1de3e32b5 Binary files /dev/null and b/examples/depth_eval_tools/example_images/depth_eval/depth16.png differ diff --git a/examples/depth_eval_tools/example_images/depth_eval/ir16.png b/examples/depth_eval_tools/example_images/depth_eval/ir16.png new file mode 100644 index 000000000..72424f728 Binary files /dev/null and b/examples/depth_eval_tools/example_images/depth_eval/ir16.png differ diff --git a/examples/depth_eval_tools/example_images/depth_eval/ir8.png b/examples/depth_eval_tools/example_images/depth_eval/ir8.png new file mode 100644 index 000000000..a1c10ea51 Binary files /dev/null and b/examples/depth_eval_tools/example_images/depth_eval/ir8.png differ diff --git a/examples/depth_eval_tools/example_images/plane_parameters/board_parameters.png b/examples/depth_eval_tools/example_images/plane_parameters/board_parameters.png new file mode 100644 index 000000000..8eebf393a Binary files /dev/null and b/examples/depth_eval_tools/example_images/plane_parameters/board_parameters.png differ diff --git a/examples/depth_eval_tools/example_images/transformation_eval/checkered_pattern.png b/examples/depth_eval_tools/example_images/transformation_eval/checkered_pattern.png new file mode 100644 index 000000000..1391fd156 Binary files /dev/null and b/examples/depth_eval_tools/example_images/transformation_eval/checkered_pattern.png differ diff --git a/examples/depth_eval_tools/example_images/transformation_eval/results.txt b/examples/depth_eval_tools/example_images/transformation_eval/results.txt new file mode 100644 index 000000000..972c466ff --- /dev/null +++ b/examples/depth_eval_tools/example_images/transformation_eval/results.txt @@ -0,0 +1 @@ + rms = 7.42723 pixels diff --git a/examples/depth_eval_tools/example_images/transformation_eval/tr_eval.png b/examples/depth_eval_tools/example_images/transformation_eval/tr_eval.png new file mode 100644 index 000000000..99fb4e889 Binary files /dev/null and b/examples/depth_eval_tools/example_images/transformation_eval/tr_eval.png differ diff --git a/examples/depth_eval_tools/example_images/transformation_eval/transformation_error.png b/examples/depth_eval_tools/example_images/transformation_eval/transformation_error.png new file mode 100644 index 000000000..5a5f319b9 Binary files /dev/null and b/examples/depth_eval_tools/example_images/transformation_eval/transformation_error.png differ diff --git a/examples/depth_eval_tools/kahelpers/inc/kahelpers.h b/examples/depth_eval_tools/kahelpers/inc/kahelpers.h new file mode 100644 index 000000000..cb83a09b3 --- /dev/null +++ b/examples/depth_eval_tools/kahelpers/inc/kahelpers.h @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#ifndef KAHELPERS_H +#define KAHELPERS_H + +#include +#include + +#include "opencv2/core.hpp" +#include + +namespace kahelpers +{ + +// to store charuco target size, number of squares, ..., etc. +struct charuco_target +{ + int squaresX; // number of chessboard squares in X direction + int squaresY; // number of chessboard squares in Y direction + float squareLength_mm; // chessboard square side length in mm + float markerLength_mm; // marker side length in mm + float marginSize_mm; // white margin around the board in mm + int aruco_dict_name; // Predefined markers dictionaries + bool valid; + charuco_target(); + // create a charuco target from a json template + charuco_target(const std::string template_file); + // read charuco board from a json template + bool read_from_json(const std::string template_file); + cv::Ptr create_board(); +}; + +// convert k4a color image to opencv mat +cv::Mat color_to_opencv(const k4a::image &im); +// convert k4a depth image to opencv mat +cv::Mat depth_to_opencv(const k4a::image &im); +// convert k4a ir image to opencv mat +cv::Mat ir_to_opencv(const k4a::image &im); +// convert opencv mat to k4a color image +k4a::image color_from_opencv(const cv::Mat &M); +// convert opencv mat to k4a depth image +k4a::image depth_from_opencv(const cv::Mat &M); +// convert opencv mat to k4a ir image +k4a::image ir_from_opencv(const cv::Mat &M); + +// calculate percentile +float cal_percentile(const cv::Mat &src, float percentile, float maxRange, int nbins = 100); +// get 8-bit gray scale image from 16-bit image. +float get_gray_gamma_img(const cv::Mat &inImg, + cv::Mat &outImg, + float gamma = 0.5f, + float maxInputValue = 5000.0, + float percentile = 99.0); + +// write calibration blob to a json file +bool write_calibration_blob(const std::vector calibration_buffer, + const std::string output_path, + const std::string calib_name); +// convert k4a calibration to opencv format i.e., camera matrix, dist coeffs +void calibration_to_opencv(const k4a_calibration_camera_t &camera_calibration, + cv::Mat &camera_matrix, + cv::Mat &dist_coeffs); +// convert k4a calibration to opencv format i.e., camera matrix, dist coeffs +// and write results to a yml file +void write_opencv_calib(const k4a_calibration_camera_t &camera_calibration, + const std::string output_path, + const std::string calib_name); + +// get ir, depth and color images from a playback +// this can be used to get single image or the mean of all images in the playback +void get_images(k4a::playback &playback, + int timestamp, + cv::Mat &ir16, + cv::Mat &depth16, + cv::Mat &color8, + bool single = false, + bool get_ir = true, + bool get_depth = true, + bool get_color = true); + +// interpolate depth value using bilinear interpolation +bool interpolate_depth(const cv::Mat &D, float x, float y, float &v); + +// create xy table that stores the normalized coordinates (xn, yn) +void create_xy_table(const k4a_calibration_t &calibration, k4a::image &xy_table); +// write the xy table to two csv files "_x.csv" and "_y.csv" +void write_xy_table(const k4a::image &xy_table, const std::string output_dir, const std::string table_name); + +// generate a checkered pattern from two imput images +bool gen_checkered_pattern(const cv::Mat &A, const cv::Mat &B, cv::Mat &C, int n = 11); + +// detect charuco markers and corners +void detect_charuco(const cv::Mat &img, + const cv::Ptr &board, + const cv::Ptr ¶ms, + std::vector &markerIds, + std::vector> &markerCorners, + std::vector &charucoIds, + std::vector &charucoCorners, + bool show_results = false); + +// find common markers between two sets +void find_common_markers(const std::vector &id1, + const std::vector &corners1, + const std::vector &id2, + const std::vector &corners2, + std::vector &cid, + std::vector &ccorners1, + std::vector &ccorners2); + +// generate markers 3d positions. Same as getBoardObjectAndImagePoints but for Charuco +void get_board_object_points_charuco(const cv::Ptr &board, + const std::vector &ids, + std::vector &corners3d); +} // namespace kahelpers +#endif \ No newline at end of file diff --git a/examples/depth_eval_tools/kahelpers/kahelpers.cpp b/examples/depth_eval_tools/kahelpers/kahelpers.cpp new file mode 100644 index 000000000..6d971798e --- /dev/null +++ b/examples/depth_eval_tools/kahelpers/kahelpers.cpp @@ -0,0 +1,635 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include // std::cout +#include // std::ofstream + +#include +#include +#include +#include +#include + +#include "inc/kahelpers.h" + +namespace kahelpers +{ + +charuco_target::charuco_target() +{ + valid = false; + squaresX = 0; + squaresY = 0; + squareLength_mm = 0.0f; + markerLength_mm = 0.0f; + marginSize_mm = 0.0f; + aruco_dict_name = 0; +} +charuco_target::charuco_target(const std::string template_file) +{ + read_from_json(template_file); +} +// read charuco board from a json file +bool charuco_target::read_from_json(const std::string template_file) +{ + valid = false; + // std::string template_file = "C:/data/k4a/test_charuco/plane.json"; + cv::FileStorage fs(template_file, cv::FileStorage::READ); + + if (!fs.isOpened()) + { + std::cerr << "failed to open " << template_file << std::endl; + return false; + } + + std::string target_name; + // second method: use FileNode::operator >> + fs["target"] >> target_name; + + cv::FileNode shapes = fs["shapes"]; + + if (shapes.type() == cv::FileNode::SEQ) + { + for (unsigned long i = 0; i < shapes.size(); i++) + { + // cv::FileNode val1 = shapes[i]["shape"]; // we only want the content of val1 + // int j = 0; + // std::cout << val1[j]; + std::string s; + // second method: use FileNode::operator >> + shapes[(int)i]["shape"] >> s; + if (s.compare("charuco") == 0) + { + if (!shapes[(int)i]["squares_x"].empty()) + squaresX = (int)shapes[(int)i]["squares_x"]; + if (!shapes[(int)i]["squares_y"].empty()) + squaresY = (int)shapes[(int)i]["squares_y"]; + if (!shapes[(int)i]["square_length"].empty()) + squareLength_mm = (float)shapes[(int)i]["square_length"]; + if (!shapes[(int)i]["marker_length"].empty()) + markerLength_mm = (float)shapes[(int)i]["marker_length"]; + if (!shapes[(int)i]["margin_size"].empty()) + marginSize_mm = (float)shapes[(int)i]["margin_size"]; + if (!shapes[(int)i]["aruco_dict_name"].empty()) + aruco_dict_name = (int)shapes[(int)i]["aruco_dict_name"]; + valid = true; + break; + } + } + } + fs.release(); + return valid; +} +cv::Ptr charuco_target::create_board() +{ + // cv::aruco::PREDEFINED_DICTIONARY_NAME dict_name = (cv::aruco::PREDEFINED_DICTIONARY_NAME)charuco.aruco_dict_name; + + cv::Ptr dictionary = cv::aruco::getPredefinedDictionary(aruco_dict_name); + cv::Ptr board = cv::aruco::CharucoBoard::create(squaresX, + squaresY, + squareLength_mm / 1000.0f, + markerLength_mm / 1000.0f, + dictionary); + return board; +} + +float cal_percentile(const cv::Mat &src, float percentile, float maxRange, int nbins) +{ + + int histSize[] = { nbins }; + float hranges[] = { 0, maxRange }; + const float *ranges[] = { hranges }; + + cv::Mat hist; + // we compute the histogram from the 0-th and 1-st channels + int channels[] = { 0 }; + cv::calcHist(&src, + 1, + channels, + cv::Mat(), // do not use mask + hist, + 1, + histSize, + ranges); + + float sum = 0; + int np = src.cols * src.rows; + int binIdx; + for (binIdx = 0; binIdx < nbins; binIdx++) + { + if ((100 * sum / np) >= percentile) + { + break; + } + sum += hist.at(binIdx); + } + float v = binIdx * maxRange / nbins; + return v; +} + +cv::Mat color_to_opencv(const k4a::image &im) +{ + // return (3 channels) + cv::Mat M3; + if (im.get_format() == k4a_image_format_t::K4A_IMAGE_FORMAT_COLOR_MJPG) + { + // turboJPEG and cv::imdecode produce slightly different results + + std::vector buffer(im.get_buffer(), im.get_buffer() + im.get_size()); + M3 = cv::imdecode(buffer, cv::IMREAD_ANYCOLOR); + } + else if (im.get_format() == k4a_image_format_t::K4A_IMAGE_FORMAT_COLOR_BGRA32) + { + cv::Mat M4(im.get_height_pixels(), im.get_width_pixels(), CV_8UC4, (void *)im.get_buffer()); + cv::cvtColor(M4, M3, cv::COLOR_BGRA2BGR); + } + else + { + std::cerr << "\nThis version supports only COLOR_MJPG and COLOR_BGRA32 format\n"; + M3 = cv::Mat(); + } + + return M3; +} + +cv::Mat depth_to_opencv(const k4a::image &im) +{ + return cv::Mat(im.get_height_pixels(), + im.get_width_pixels(), + CV_16U, + (void *)im.get_buffer(), + static_cast(im.get_stride_bytes())) + .clone(); +} + +cv::Mat ir_to_opencv(const k4a::image &im) +{ + return depth_to_opencv(im); +} + +k4a::image color_from_opencv(const cv::Mat &M) +{ + cv::Mat M4; + if (M.type() == CV_8UC4) + M4 = M; + else if (M.type() == CV_8UC3) + cv::cvtColor(M, M4, cv::COLOR_BGR2BGRA); + else if (M.type() == CV_8UC1) + cv::cvtColor(M, M4, cv::COLOR_GRAY2BGRA); + else // (M.type() != CV_8UC4 && M.type() != CV_8UC3 && M.type() != CV_8UC1) + { + std::cerr << "Only CV_8UC4, CV_8UC3 and CV_8UC1 are supported \n"; + return NULL; + } + + k4a::image img = k4a::image::create(K4A_IMAGE_FORMAT_COLOR_BGRA32, M4.cols, M4.rows, static_cast(M4.step)); + memcpy(img.get_buffer(), M4.data, static_cast(M4.total() * M4.elemSize())); + + return img; +} + +k4a::image depth_from_opencv(const cv::Mat &M) +{ + + if (M.type() != CV_16U) + { + std::cerr << "Only CV_16U is supported \n"; + return NULL; + } + k4a::image img = k4a::image::create(K4A_IMAGE_FORMAT_DEPTH16, M.cols, M.rows, static_cast(M.step)); + memcpy((void *)img.get_buffer(), M.data, static_cast(M.total() * M.elemSize())); + + return img; +} + +k4a::image ir_from_opencv(const cv::Mat &M) +{ + + if (M.type() != CV_16U) + { + std::cerr << "Only CV_16U is supported \n"; + return NULL; + } + k4a::image img = k4a::image::create(K4A_IMAGE_FORMAT_IR16, M.cols, M.rows, static_cast(M.step)); + memcpy(img.get_buffer(), M.data, static_cast(M.total() * M.elemSize())); + + return img; +} + +float get_gray_gamma_img(const cv::Mat &inImg, cv::Mat &outImg, float gamma, float maxInputValue, float percentile) +{ + cv::Mat floatMat; + inImg.convertTo(floatMat, CV_32F); + // floatMat = abs(floatMat) + 1; + double minVal, maxVal; + cv::minMaxLoc(floatMat, &minVal, &maxVal); + maxInputValue = cv::min(maxInputValue, (float)maxVal); + + float v = cal_percentile(floatMat, percentile, maxInputValue + 1, 1000); + + float scale = 255.0f / static_cast(pow(v, gamma)); // gamma + + floatMat = cv::max(floatMat, 1); + // cv::log(floatMat, logImg); + cv::pow(floatMat, gamma, floatMat); + // 8.8969e-11 -1.9062e-06 0.0191 12.6882 + + // cv::Mat(min(scale*floatMat, 255)).convertTo(outImg, CV_8UC1); + cv::Mat(scale * floatMat).convertTo(outImg, CV_8UC1); + return scale; +} + +bool write_calibration_blob(const std::vector calibration_buffer, + const std::string output_path, + const std::string calib_name) +{ + std::string file_name = output_path + "\\" + calib_name + ".json"; + std::ofstream ofs(file_name, std::ofstream::binary); + if (!ofs.is_open()) + { + std::cout << "Error opening file"; + return false; + } + + ofs.write(reinterpret_cast(&calibration_buffer[0]), + static_cast(calibration_buffer.size() - 1)); + ofs.close(); + + return true; +} + +void calibration_to_opencv(const k4a_calibration_camera_t &camera_calibration, + cv::Mat &camera_matrix, + cv::Mat &dist_coeffs) +{ + k4a_calibration_intrinsic_parameters_t intrinsics = camera_calibration.intrinsics.parameters; + + std::vector _camera_matrix = + { intrinsics.param.fx, 0.f, intrinsics.param.cx, 0.f, intrinsics.param.fy, intrinsics.param.cy, 0.f, 0.f, 1.f }; + camera_matrix = cv::Mat(3, 3, CV_32F, &_camera_matrix[0]).clone(); + std::vector _dist_coeffs = { intrinsics.param.k1, intrinsics.param.k2, intrinsics.param.p1, + intrinsics.param.p2, intrinsics.param.k3, intrinsics.param.k4, + intrinsics.param.k5, intrinsics.param.k6 }; + dist_coeffs = cv::Mat(8, 1, CV_32F, &_dist_coeffs[0]).clone(); +} + +void write_opencv_calib(const k4a_calibration_camera_t &camera_calibration, + const std::string output_path, + const std::string calib_name) +{ + cv::Mat camera_matrix; + cv::Mat dist_coeffs; + calibration_to_opencv(camera_calibration, camera_matrix, dist_coeffs); + // save opencv cal + std::string file_name = output_path + "\\" + calib_name + ".yml"; + cv::FileStorage fs(file_name.c_str(), cv::FileStorage::WRITE); + int s[2] = { camera_calibration.resolution_width, camera_calibration.resolution_height }; + cv::Mat image_size(2, 1, CV_32S, s); + + fs << "K" << camera_matrix; + fs << "dist" << dist_coeffs; + fs << "img_size" << image_size; + fs.release(); +} + +void get_images(k4a::playback &playback, + int timestamp, + cv::Mat &ir16, + cv::Mat &depth16, + cv::Mat &color8, + bool single, + bool get_ir, + bool get_depth, + bool get_color) +{ + playback.seek_timestamp(std::chrono::microseconds(timestamp * 1000), K4A_PLAYBACK_SEEK_BEGIN); + + // std::chrono::microseconds length = playback.get_recording_length(); + // printf("Seeking to timestamp: %d/%d (ms)\n", timestamp, (int)(length.count() / 1000)); + + int n_ir = 0, n_depth = 0, n_color = 0; + cv::Mat mean_ir, mean_depth; + cv::Mat mean_color[3]; + + k4a::capture cap; + int count = 0; + while (playback.get_next_capture(&cap)) + { + std::cout << "Processing frame # " << count++ << std::endl; + + k4a::image depth_img; + k4a::image ir_img; + k4a::image color_img; + + if (get_depth) + depth_img = cap.get_depth_image(); + if (get_ir) + ir_img = cap.get_ir_image(); + if (get_color) + color_img = cap.get_color_image(); + + if (ir_img.is_valid()) + { + cv::Mat frame32; + ir_to_opencv(ir_img).convertTo(frame32, CV_32F); + if (n_ir < 1) + mean_ir = frame32.clone(); + else + mean_ir += frame32.clone(); + n_ir++; + } + if (depth_img.is_valid()) + { + cv::Mat frame32; + depth_to_opencv(depth_img).convertTo(frame32, CV_32F); + if (n_depth < 1) + mean_depth = frame32.clone(); + else + mean_depth += frame32.clone(); + n_depth++; + } + if (color_img.is_valid()) + { + cv::Mat frame32[3]; + cv::Mat bands[3]; + cv::split(color_to_opencv(color_img), bands); + for (int ch = 0; ch < 3; ch++) + { + bands[ch].convertTo(frame32[ch], CV_32F); + if (n_color < 1) + mean_color[ch] = frame32[ch].clone(); + else + mean_color[ch] += frame32[ch].clone(); + } + n_color++; + } + // get single frame + if (single) + break; + } + if (n_ir > 1) + mean_ir /= n_ir; + if (n_depth > 1) + mean_depth /= n_depth; + if (n_color > 1) + { + for (int ch = 0; ch < 3; ch++) + { + mean_color[ch] /= n_color; + } + } + + if (!mean_ir.empty()) + mean_ir.convertTo(ir16, CV_16U); + if (!mean_depth.empty()) + mean_depth.convertTo(depth16, CV_16U); + if (!mean_color[0].empty()) + { + cv::Mat channels[3]; + for (int ch = 0; ch < 3; ch++) + { + mean_color[ch].convertTo(channels[ch], CV_8U); + } + cv::merge(channels, 3, color8); + } +} + +bool interpolate_depth(const cv::Mat &D, float x, float y, float &v) +{ + int xi = (int)x; + int yi = (int)y; + + if (xi < 1 || xi >= (D.cols - 1) || yi < 1 || yi >= (D.rows - 1)) + return false; + + float a = (float)D.at(yi, xi); + float b = (float)D.at(yi, xi + 1); + float c = (float)D.at(yi + 1, xi); + float d = (float)D.at(yi + 1, xi + 1); + + float s = x - xi; + float t = y - yi; + + float v1 = (1.0f - s) * a + s * b; + float v2 = (1.0f - s) * c + s * d; + v = (1.0f - t) * v1 + t * v2; + + return true; +} + +void create_xy_table(const k4a_calibration_t &calibration, k4a::image &xy_table) +{ + k4a_float2_t *table_data = (k4a_float2_t *)(void *)xy_table.get_buffer(); + + int width = calibration.depth_camera_calibration.resolution_width; + int height = calibration.depth_camera_calibration.resolution_height; + + k4a_float2_t p; + k4a_float3_t ray; + int valid; + + for (int y = 0, idx = 0; y < height; y++) + { + p.xy.y = (float)y; + for (int x = 0; x < width; x++, idx++) + { + p.xy.x = (float)x; + + k4a_calibration_2d_to_3d( + &calibration, &p, 1.f, K4A_CALIBRATION_TYPE_DEPTH, K4A_CALIBRATION_TYPE_DEPTH, &ray, &valid); + + if (valid) + { + table_data[idx].xy.x = ray.xyz.x; + table_data[idx].xy.y = ray.xyz.y; + } + else + { + table_data[idx].xy.x = nanf(""); + table_data[idx].xy.y = nanf(""); + } + } + } +} + +void write_xy_table(const k4a::image &xy_table, const std::string output_dir, const std::string table_name) +{ + std::string xfile_name = output_dir + "\\" + table_name + "_x.csv"; + std::string yfile_name = output_dir + "\\" + table_name + "_y.csv"; + + k4a_float2_t *xy_table_data = (k4a_float2_t *)(void *)(xy_table.get_buffer()); + + int width = xy_table.get_width_pixels(); + int height = xy_table.get_height_pixels(); + + std::ofstream ofsx(xfile_name, std::ofstream::out); + std::ofstream ofsy(yfile_name, std::ofstream::out); + + if (!ofsx.is_open() || !ofsy.is_open()) + { + std::cout << "Error opening file"; + return; + } + + // xy_table. + for (int y = 0; y < height; y++) + { + for (int x = 0; x < width; x++) + { + int idx = y * width + x; + ofsx << xy_table_data[idx].xy.x; + ofsy << xy_table_data[idx].xy.y; + if (x < width - 1) + { + ofsx << ","; + ofsy << ","; + } + } + ofsx << std::endl; + ofsy << std::endl; + } + + ofsx.close(); + ofsy.close(); +} + +bool gen_checkered_pattern(const cv::Mat &A, const cv::Mat &B, cv::Mat &C, int n) +{ + // gen checkered pattern from A & B + cv::Mat Ma, Mb; + if (A.depth() != 0 || B.depth() != 0) + { + std::cerr << "A.depth()!=0 || B.depth()!=0 \n"; + return false; + } + if (A.channels() == B.channels()) + { + Ma = A; + Mb = B; + } + else if (A.channels() == 1 && B.channels() == 3) + { + cv::cvtColor(A, Ma, cv::COLOR_GRAY2BGR); + Mb = B; + } + else if (A.channels() == 3 && B.channels() == 1) + { + Ma = A; + cv::cvtColor(B, Mb, cv::COLOR_GRAY2BGR); + } + else + { + std::cerr << "number of channels is not supported \n"; + return false; + } + + if (n <= 0) + { + std::cerr << "Invalid value for n, must be greater than 0 \n"; + return false; + } + + int sx = Ma.cols / n; + int sy = Ma.rows / n; + for (int i = 0; i < n; i++) + { + for (int j = (i) % 2; j < n; j += 2) + { + Mb(cv::Rect(j * sx, i * sy, sx, sy)).copyTo(Ma(cv::Rect(j * sx, i * sy, sx, sy))); + } + } + C = Ma.clone(); + return true; +} + +void detect_charuco(const cv::Mat &img, + const cv::Ptr &board, + const cv::Ptr ¶ms, + std::vector &markerIds, + std::vector> &markerCorners, + std::vector &charucoIds, + std::vector &charucoCorners, + bool show_results) +{ + + markerIds.clear(); + markerCorners.clear(); + charucoIds.clear(); + charucoCorners.clear(); + + cv::aruco::detectMarkers(img, board->dictionary, markerCorners, markerIds, params); + if (markerIds.size() > 0) + { + cv::aruco::interpolateCornersCharuco(markerCorners, markerIds, img, board, charucoCorners, charucoIds); + + if (show_results) + { + cv::Mat img_copy, img_copy2; + if (img.depth() == CV_8U && img.channels() == 1) + cv::cvtColor(img, img_copy, cv::COLOR_GRAY2BGR); + else + img.copyTo(img_copy); + img_copy.copyTo(img_copy2); + cv::aruco::drawDetectedMarkers(img_copy, markerCorners, markerIds); + + // if at least one charuco corner detected + if (charucoIds.size() > 0) + cv::aruco::drawDetectedCornersCharuco(img_copy2, charucoCorners, charucoIds, cv::Scalar(255, 0, 0)); + + if (MAX(img.rows, img.cols) > 1024) + { + double sf = 1024. / MAX(img.rows, img.cols); + cv::resize(img_copy, img_copy, cv::Size(), sf, sf); + cv::resize(img_copy2, img_copy2, cv::Size(), sf, sf); + } + cv::imshow("aruco", img_copy); + cv::imshow("charuco", img_copy2); + cv::waitKey(0); + } + } +} + +void find_common_markers(const std::vector &id1, + const std::vector &corners1, + const std::vector &id2, + const std::vector &corners2, + std::vector &cid, + std::vector &ccorners1, + std::vector &ccorners2) +{ + cid.clear(); + ccorners1.clear(); + ccorners2.clear(); + + for (size_t i = 0; i < id1.size(); i++) + { + auto it = std::find(id2.begin(), id2.end(), id1[i]); + if (it != id2.end()) + { + size_t j = (size_t)(it - id2.begin()); + // std::cout << std::endl << v1[i] << " " << *it << " " << v2[j] << std::endl; + cid.push_back(id1[i]); + ccorners1.push_back(corners1[i]); + ccorners2.push_back(corners2[j]); + } + } + return; +} + +// generate markers 3d positions. Same as getBoardObjectAndImagePoints but for Charuco +void get_board_object_points_charuco(const cv::Ptr &board, + const std::vector &ids, + std::vector &corners3d) +{ + corners3d.clear(); + + for (size_t i = 0; i < ids.size(); i++) + { + size_t id = (size_t)ids[i]; + corners3d.push_back(board->chessboardCorners[id]); + } + return; +} + +} // namespace kahelpers \ No newline at end of file diff --git a/examples/depth_eval_tools/mkv2images/CMakeLists.txt b/examples/depth_eval_tools/mkv2images/CMakeLists.txt new file mode 100644 index 000000000..8c419cec6 --- /dev/null +++ b/examples/depth_eval_tools/mkv2images/CMakeLists.txt @@ -0,0 +1,23 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +find_package(OpenCV REQUIRED) +include_directories( + . + ./inc/ + ../kahelpers/ + ../kahelpers/inc/ + ) + +add_executable(mkv2images + mkv2images.cpp + ./inc/mkv2images.h + ../kahelpers/kahelpers.cpp + ../kahelpers/inc/kahelpers.h + ) + +target_link_libraries(mkv2images PRIVATE + k4a::k4a + k4a::k4arecord + ${OpenCV_LIBS} + ) \ No newline at end of file diff --git a/examples/depth_eval_tools/mkv2images/README.md b/examples/depth_eval_tools/mkv2images/README.md new file mode 100644 index 000000000..85a621d64 --- /dev/null +++ b/examples/depth_eval_tools/mkv2images/README.md @@ -0,0 +1,27 @@ +# Azure Kinect - Depth Evaluation Tools Examples - mkv2images + +--- +## Description + + Dump mkv file to png images. + +--- +## Usage + + ``` + ./mkv2images -h or -help or -? print the help message + ./mkv2images -in= -out= -d= -i= -c= + -f=<0:dump mean images only, 1 : dump first frame> + -gg= + -gm= + -gp= + ``` + + Example Command: ```./mkv2images -in=board1.mkv -out=c:/data -c=0 -f=0``` + +--- + +## Dependencies + + OpenCV + OpenCV Contrib diff --git a/examples/depth_eval_tools/mkv2images/inc/mkv2images.h b/examples/depth_eval_tools/mkv2images/inc/mkv2images.h new file mode 100644 index 000000000..2766abdb6 --- /dev/null +++ b/examples/depth_eval_tools/mkv2images/inc/mkv2images.h @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#ifndef MKV2IMAGES_H +#define MKV2IMAGES_H + +#include + +void help(); + +std::string extract_filename(const std::string &full_filename); + +#endif diff --git a/examples/depth_eval_tools/mkv2images/mkv2images.cpp b/examples/depth_eval_tools/mkv2images/mkv2images.cpp new file mode 100644 index 000000000..61a8dc32d --- /dev/null +++ b/examples/depth_eval_tools/mkv2images/mkv2images.cpp @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include // std::cout +#include // std::ofstream + +#include +#include + +#include +#include +#include +#include +#include + +#include "inc/mkv2images.h" +#include "kahelpers.h" + +using namespace kahelpers; + +void help() +{ + std::cout << "\nDump mkv as png images.\n"; + std::cout + << "Usage:\n" + "./mkv2images -h or -help or -? print this help message\n" + "./mkv2images -in= -out= -d= -i= -c=" + " -f=<0:dump mean images only, 1 : dump first frame>\n" + " -gg=\n" + " -gm=\n" + " -gp=\n" + " Example:\n" + "./mkv2images -in=board1.mkv -out=c:/data -c=0 -f=0\n"; +} + +// extract filename from the full path +std::string extract_filename(const std::string &full_filename) +{ + unsigned long index = (unsigned long)full_filename.find_last_of("/\\"); + std::string filename = full_filename.substr(index + 1); + return filename; +} + +int main(int argc, char **argv) +{ + + cv::CommandLineParser parser(argc, + argv, + "{help h usage ?| |print this message}" + "{in| | full path of the wfov_binned mkv file}" + "{out| | full path of the output dir}" + "{d|1| dump depth}" + "{i|1| dump ir}" + "{c|1| dump color}" + "{f|0| 0:dump mean images only, 1:dump first frame}" + "{gg|0.5| gray_gamma used to convert ir data to 8bit gray image}" + "{gm|4000.0| gray_max used to convert ir data to 8bit gray image}" + "{gp|99.0| percentile used to convert ir data to 8bit gray image}"); + // get input argument + + std::string depth_mkv = parser.get("in"); + std::string output_dir = parser.get("out"); + bool dump_depth = parser.get("d") > 0; + bool dump_ir = parser.get("i") > 0; + bool dump_color = parser.get("c") > 0; + int f = parser.get("f"); + float gray_gamma = parser.get("gg"); + float gray_max = parser.get("gm"); + float gray_percentile = parser.get("gp"); + + if (depth_mkv.empty() || output_dir.empty()) + { + help(); + return 1; + } + + if (parser.has("help")) + { + help(); + return 0; + } + + // Timestamp in milliseconds. Defaults to 1 sec as the first couple frames don't contain color + int timestamp = 1000; + + k4a::playback playback = k4a::playback::open(depth_mkv.c_str()); + + cv::Mat ir16, depth16, color8; + // get all images + get_images(playback, timestamp, ir16, depth16, color8, f, dump_ir, dump_depth, dump_color); + + cv::Mat ir8; + if (!ir16.empty()) + get_gray_gamma_img(ir16, ir8, gray_gamma, gray_max, gray_percentile); + + // save images as png files + std::string filename = extract_filename(depth_mkv); + if (!ir16.empty()) + cv::imwrite(output_dir + "/" + filename + "-ir16.png", ir16); + if (!depth16.empty()) + cv::imwrite(output_dir + "/" + filename + "-depth16.png", depth16); + if (!color8.empty()) + cv::imwrite(output_dir + "/" + filename + "-color8.png", color8); + if (!ir8.empty()) + cv::imwrite(output_dir + "/" + filename + "-ir8.png", ir8); + + return 0; +} diff --git a/examples/depth_eval_tools/plane_files/plane.json b/examples/depth_eval_tools/plane_files/plane.json new file mode 100644 index 000000000..f1acb36e0 --- /dev/null +++ b/examples/depth_eval_tools/plane_files/plane.json @@ -0,0 +1,20 @@ +{ +"target": "charuco1", +"type": "charuco", +"shapes": [ + { + "shape": "charuco", + "squares_x": 14, + "squares_y": 9, + "square_length": 40, + "marker_length": 30, + "margin_size": 20, + "width": 600, + "height": 400, + "x": -20, + "y": -20, + "dpi_factor": 5, + "aruco_dict_name": 6 +} +] +} \ No newline at end of file diff --git a/examples/depth_eval_tools/plane_files/plane.pdf b/examples/depth_eval_tools/plane_files/plane.pdf new file mode 100644 index 000000000..7ca5048f2 Binary files /dev/null and b/examples/depth_eval_tools/plane_files/plane.pdf differ diff --git a/examples/depth_eval_tools/plane_files/plane_large.json b/examples/depth_eval_tools/plane_files/plane_large.json new file mode 100644 index 000000000..987f785c4 --- /dev/null +++ b/examples/depth_eval_tools/plane_files/plane_large.json @@ -0,0 +1,20 @@ +{ +"target": "charuco_large", +"type": "charuco", +"shapes": [ +{ +"shape": "charuco", +"squares_x": 14, +"squares_y": 9, +"square_length": 80, +"marker_length": 60, +"margin_size": 40, +"width": 1200, +"height": 800, +"x": -40, +"y": -40, +"dpi_factor": 5, +"aruco_dict_name": 6 +} +] +} \ No newline at end of file diff --git a/examples/depth_eval_tools/plane_files/plane_large.pdf b/examples/depth_eval_tools/plane_files/plane_large.pdf new file mode 100644 index 000000000..9bf8558cc Binary files /dev/null and b/examples/depth_eval_tools/plane_files/plane_large.pdf differ diff --git a/examples/depth_eval_tools/plane_files/plane_small.json b/examples/depth_eval_tools/plane_files/plane_small.json new file mode 100644 index 000000000..608b73db3 --- /dev/null +++ b/examples/depth_eval_tools/plane_files/plane_small.json @@ -0,0 +1,20 @@ +{ +"target": "charuco_small", +"type": "charuco", +"shapes": [ +{ +"shape": "charuco", +"squares_x": 10, +"squares_y": 7, +"square_length": 25, +"marker_length": 19, +"margin_size": 12, +"width": 274, +"height": 199, +"x": -12, +"y": -12, +"dpi_factor": 5, +"aruco_dict_name": 5 +} +] +} \ No newline at end of file diff --git a/examples/depth_eval_tools/plane_files/plane_small.pdf b/examples/depth_eval_tools/plane_files/plane_small.pdf new file mode 100644 index 000000000..a2840a4f6 Binary files /dev/null and b/examples/depth_eval_tools/plane_files/plane_small.pdf differ diff --git a/examples/depth_eval_tools/transformation_eval/CMakeLists.txt b/examples/depth_eval_tools/transformation_eval/CMakeLists.txt new file mode 100644 index 000000000..e3f15f64d --- /dev/null +++ b/examples/depth_eval_tools/transformation_eval/CMakeLists.txt @@ -0,0 +1,23 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +find_package(OpenCV REQUIRED) +include_directories( + . + ./inc/ + ../kahelpers/ + ../kahelpers/inc/ + ) + +add_executable(transformation_eval + transformation_eval.cpp + ./inc/transformation_eval.h + ../kahelpers/kahelpers.cpp + ../kahelpers/inc/kahelpers.h + ) + +target_link_libraries(transformation_eval PRIVATE + k4a::k4a + k4a::k4arecord + ${OpenCV_LIBS} + ) \ No newline at end of file diff --git a/examples/depth_eval_tools/transformation_eval/README.md b/examples/depth_eval_tools/transformation_eval/README.md new file mode 100644 index 000000000..5133191e0 --- /dev/null +++ b/examples/depth_eval_tools/transformation_eval/README.md @@ -0,0 +1,37 @@ +# Azure Kinect - Depth Evaluation Tools Examples - transformation_eval + +--- + +## Description + + Transformation Evaluation Tool for K4A. + + This tool utilizes two mkv files. + + The 1st mkv file is PASSIVE_IR recorded using: ```k4arecorder.exe -c 3072p -d PASSIVE_IR -l 3 board1.mkv``` + + The 2nd mkv file is WFOV_2X2BINNED recorded using: ```k4arecorder.exe -c 3072p -d WFOV_2X2BINNED -l 3 board2.mkv``` + + This version supports WFOV_2X2BINNED but can be easily generalized. + +--- + +## Usage + + ``` + ./transformation_eval -h or -help or -? print the help message + ./transformation_eval -i= -d= -t= + -out= -s=<1:generate and save result images> + -gg= + -gm= + -gp= + ``` + + Example Command: ```./transformation_eval -i=board1.mkv -d=board2.mkv -t=plane.json -out=c:/data``` + +--- + +## Dependencies + + OpenCV + OpenCV Contrib diff --git a/examples/depth_eval_tools/transformation_eval/inc/transformation_eval.h b/examples/depth_eval_tools/transformation_eval/inc/transformation_eval.h new file mode 100644 index 000000000..705704473 --- /dev/null +++ b/examples/depth_eval_tools/transformation_eval/inc/transformation_eval.h @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#ifndef TRANSFORMATION_EVAL_H +#define TRANSFORMATION_EVAL_H + +#include + +#include +#include + +void help(); + +int calculate_transformation_error(const cv::Mat &depth16, + const cv::Mat &color8, + const std::vector &corners_d, + const std::vector &corners_c, + const k4a::calibration &calibration, + float &rms, + cv::Mat &err_img, + bool gen_err_img = true); + +static bool process_mkv(const std::string &passive_ir_mkv, + const std::string &depth_mkv, + const std::string &template_file, + int timestamp, + const std::string &output_dir, + float gray_gamma, + float gray_max, + float gray_percentile, + bool save_images); + +#endif diff --git a/examples/depth_eval_tools/transformation_eval/transformation_eval.cpp b/examples/depth_eval_tools/transformation_eval/transformation_eval.cpp new file mode 100644 index 000000000..91985a1b0 --- /dev/null +++ b/examples/depth_eval_tools/transformation_eval/transformation_eval.cpp @@ -0,0 +1,319 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include // std::cout +#include // std::ofstream + +#include +#include + +#include +#include +#include +#include +#include + +#include "inc/transformation_eval.h" +#include "kahelpers.h" + +using namespace kahelpers; + +void help() +{ + std::cout << "\nTransformation Evaluation Tool for K4A.\n"; + std::cout << "\nit uses 2 mkv files:\n"; + std::cout + << "\t 1st is PASSIVE_IR recorded using: \n\t\t k4arecorder.exe -c 3072p -d PASSIVE_IR -l 3 board1.mkv\n"; + std::cout << "\t 2nd is WFOV_2X2BINNED recorded using: \n\t\t k4arecorder.exe -c 3072p -d WFOV_2X2BINNED -l 3 " + "board2.mkv\n"; + std::cout << "\t This version supports WFOV_2X2BINNED but can be easily generalized\n"; + std::cout << "Usage:\n" + "./transformation_eval -h or -help or -? print this help message\n" + "./transformation_eval -i= -d= -t=" + " -out= -s=<1:generate and save result images>\n" + " -gg=\n" + " -gm=\n" + " -gp=\n" + " Example:\n" + "./transformation_eval -i=board1.mkv -d=board2.mkv -t=plane.json -out=c:/data\n"; +} + +// map markers from depth image to color image and calculate the reprojection error rms +int calculate_transformation_error(const cv::Mat &depth16, + const cv::Mat &color8, + const std::vector &corners_d, + const std::vector &corners_c, + const k4a::calibration &calibration, + float &rms, + cv::Mat &err_img, + bool gen_err_img) +{ + rms = 0; + int nValid = 0; + + cv::Mat xpc_predict((int)corners_c.size(), 2, CV_32F, -1.0); + for (unsigned long i = 0; i < corners_d.size(); i++) + { + float d_mm; + interpolate_depth(depth16, corners_d[i].x, corners_d[i].y, d_mm); + if (d_mm <= 0) + continue; + + // 1. given detections in depth and color corners_d & corners_c + k4a_float2_t pd, pc; + pd.v[0] = corners_d[i].x; + pd.v[1] = corners_d[i].y; + + // 2. map pixel from depth image to color image + bool valid = + calibration.convert_2d_to_2d(pd, d_mm, K4A_CALIBRATION_TYPE_DEPTH, K4A_CALIBRATION_TYPE_COLOR, &pc); + // 3. if valid, calculate the reprojection error between: + // a) prediction: transformed markers from depth to color using intrinsics of both cameras and R&T between + // them + // b) detection: detected markers in color + // err = prediction - detection + if (valid) + { + xpc_predict.at((int)i, 0) = pc.v[0]; + xpc_predict.at((int)i, 1) = pc.v[1]; + float dx = pc.v[0] - corners_c[i].x; + float dy = pc.v[1] - corners_c[i].y; + rms += dx * dx + dy * dy; + nValid++; + } + } + + if (nValid > 0) + rms = (float)sqrt(rms / nValid); + + // show the error on the color image + if (nValid > 0 && gen_err_img) + { + color8.copyTo(err_img); + for (int i = 0; i < xpc_predict.rows; i++) + { + if (xpc_predict.at(i, 0) >= 0) // valid + { + cv::Point2f p1(corners_c[(unsigned long)i].x, corners_c[(unsigned long)i].y); + cv::Point2f p2(xpc_predict.at(i, 0), xpc_predict.at(i, 1)); + + cv::line(err_img, p1, p2, cv::Scalar(0, 0, 255), 1); + cv::drawMarker(err_img, p1, cv::Scalar(0, 255, 0), cv::MARKER_CROSS, 2); + cv::drawMarker(err_img, p2, cv::Scalar(255, 0, 0), cv::MARKER_CROSS, 2); + } + } + } + + return nValid; +} + +// Timestamp in milliseconds. Defaults to 1 sec as the first couple frames don't contain color +static bool process_mkv(const std::string &passive_ir_mkv, + const std::string &depth_mkv, + const std::string &template_file, + int timestamp, + const std::string &output_dir, + float gray_gamma, + float gray_max, + float gray_percentile, + bool save_images) +{ + + // + // get passive ir + k4a::playback playback_ir = k4a::playback::open(passive_ir_mkv.c_str()); + if (playback_ir.get_calibration().depth_mode != K4A_DEPTH_MODE_PASSIVE_IR) + { + std::cerr << "depth_mode != K4A_DEPTH_MODE_PASSIVE_IR"; + return false; + } + cv::Mat passive_ir; + cv::Mat nullMat; + get_images(playback_ir, timestamp, passive_ir, nullMat, nullMat, false, true, false, false); + // generate a 8bit gray scale image from the passive ir so it can be used for marker detection + cv::Mat passive_ir8; + get_gray_gamma_img(passive_ir, passive_ir8, gray_gamma, gray_max, gray_percentile); + + // + // get depth & color + k4a::playback playback = k4a::playback::open(depth_mkv.c_str()); + // K4A_DEPTH_MODE_WFOV_2X2BINNED : Depth captured at 512x512 + // this version of the code supports WFOV_2X2BINNED but can be easily generalized, see switch statement below + k4a_depth_mode_t depth_mode = playback.get_calibration().depth_mode; + if (depth_mode != K4A_DEPTH_MODE_WFOV_2X2BINNED) + { + std::cerr << "depth_mode != K4A_DEPTH_MODE_WFOV_2X2BINNED"; + return false; + } + + cv::Mat ir16, depth16, color8; + // getting mean depth, ir and color (single=false) + get_images(playback, timestamp, ir16, depth16, color8, false, true, true, true); + // generate a 8bit gray scale image from ir + cv::Mat ir8; + get_gray_gamma_img(ir16, ir8, gray_gamma, gray_max, gray_percentile); + + // + // create a charuco target from a json template + charuco_target charuco(template_file); + cv::Ptr board = charuco.create_board(); + + // + // detect markers in passive_ir8 + cv::Ptr params = cv::aruco::DetectorParameters::create(); + params->cornerRefinementMethod = cv::aruco::CornerRefineMethod::CORNER_REFINE_NONE; // best option as my limited + // testing indicated + + std::vector markerIds_ir; + std::vector> markerCorners_ir; + std::vector charucoIds_ir; + std::vector charucoCorners_ir; + + detect_charuco(passive_ir8, board, params, markerIds_ir, markerCorners_ir, charucoIds_ir, charucoCorners_ir, false); + // + // detect markers in color + std::vector markerIds_color; + std::vector> markerCorners_color; + std::vector charucoIds_color; + std::vector charucoCorners_color; + + detect_charuco( + color8, board, params, markerIds_color, markerCorners_color, charucoIds_color, charucoCorners_color, false); + + std::vector common_id; + std::vector common_corners_ir; + std::vector common_corners_color; + + // find common markers between ir and color sets + find_common_markers(charucoIds_ir, + charucoCorners_ir, + charucoIds_color, + charucoCorners_color, + common_id, + common_corners_ir, + common_corners_color); + + std::cout << "\n board has " << board->chessboardCorners.size() << " charuco corners"; + std::cout << "\n corners detected in ir = " << charucoIds_ir.size(); + std::cout << "\n corners detected in color = " << charucoIds_color.size(); + std::cout << "\n number of common corners = " << common_id.size(); + + // convert opencv mat back to k4a images + k4a::image ir_img, depth_img, color_img; + if (!ir16.empty()) + ir_img = ir_from_opencv(ir16); + if (!depth16.empty()) + depth_img = depth_from_opencv(depth16); + if (!color8.empty()) + color_img = color_from_opencv(color8); + + k4a::calibration calibration = playback.get_calibration(); + k4a::transformation transformation(calibration); + + if (save_images) + { + // transform color image into depth camera geometry + k4a::image transformed_color = transformation.color_image_to_depth_camera(depth_img, color_img); + cv::Mat color8t = color_to_opencv(transformed_color); + cv::Mat mixM; + if (gen_checkered_pattern(ir8, color8t, mixM, 17)) + cv::imwrite(output_dir + "/" + "checkered_pattern.png", mixM); + } + + std::vector common_corners_d; + // 1024 --> 512 + common_corners_d = common_corners_ir; + for (size_t i = 0; i < common_corners_d.size(); i++) + { + // modify this to support other depth modes + switch (depth_mode) + { + case K4A_DEPTH_MODE_WFOV_2X2BINNED: + // for passive ir res (1024) to depth res (512) + common_corners_d[i].x /= 2; + common_corners_d[i].y /= 2; + break; + default: + common_corners_d[i].x /= 2; + common_corners_d[i].y /= 2; + } + } + + float rms = 1e6; + cv::Mat err_img; + int n_valid = calculate_transformation_error( + depth16, color8, common_corners_d, common_corners_color, calibration, rms, err_img, save_images); + + if (n_valid > 0) + { + std::cout << std::endl << " rms = " << rms << " pixels" << std::endl; + if (save_images) + cv::imwrite(output_dir + "/" + "transformation_error.png", err_img); + } + + std::string file_name = output_dir + "/" + "results.txt"; + std::ofstream ofs(file_name); + if (!ofs.is_open()) + std::cerr << "Error opening " << file_name; + else + { + ofs << " rms = " << rms << " pixels" << std::endl; + ofs.close(); + } + + return true; +} + +int main(int argc, char **argv) +{ + + cv::CommandLineParser parser(argc, + argv, + "{help h usage ?| |print this message}" + "{i| | full path of the passive_ir mkv file}" + "{d| | full path of the wfov_binned mkv file}" + "{t| | full path of the board json file e.g., /plane.json}" + "{out| | full path of the output dir}" + "{s|1| generate and save result images}" + "{gg|0.5| gray_gamma used to convert ir data to 8bit gray image}" + "{gm|4000.0| gray_max used to convert ir data to 8bit gray image}" + "{gp|99.0| percentile used to convert ir data to 8bit gray image}"); + // get input argument + std::string passive_ir_mkv = parser.get("i"); + std::string depth_mkv = parser.get("d"); + std::string template_file = parser.get("t"); + std::string output_dir = parser.get("out"); + + if (passive_ir_mkv.empty() || depth_mkv.empty() || template_file.empty() || output_dir.empty()) + { + help(); + return 1; + } + bool save_images = parser.get("s") > 0; + + float gray_gamma = parser.get("gg"); + float gray_max = parser.get("gm"); + float gray_percentile = parser.get("gp"); + + if (parser.has("help")) + { + help(); + return 0; + } + + // Timestamp in milliseconds. Defaults to 1 sec as the first couple frames don't contain color + int timestamp = 1000; + + bool stat = process_mkv(passive_ir_mkv, + depth_mkv, + template_file, + timestamp, + output_dir, + gray_gamma, + gray_max, + gray_percentile, + save_images); + + return (stat) ? 0 : 1; +}