cl-stitch: support 4k resolution mode
* tune 4k parameters for video stream
* add debug information to make it easier for qualiy tuning
* support framerate options in test-image-stitching via OpenCV codec
* gst-launch-1.0 cmdline:
$ gst-launch-1.0 filesrc location=input.nv12 \
! videoparse format=nv12 width=4096 height=2048 framerate=24/1 \
! xcamfilter copy-mode=1 enable-stitch=true stitch-scale=local \
stitch-fisheye-map=true stitch-fm-ocl=true stitch-res-mode=4k \
! video/x-raw, foramt=NV12, width=4096, height=2048 \
! queue ! vaapih264enc rate-control=cbr \
! tcpclientsink host="host-ip" port=3000 blocksize=1024000 sync=false
* test-image-stitching cmdline:
$ test-image-stitching --input input.nv12 --output output.mp4 \
--input-w 4096 --input-h 2048 --output-w 4096 --output-h 2048 \
--scale-mode local --enable-fisheyemap --fm-ocl true \
--res-mode 4k --framerate 24.0 --save true
Signed-off-by: Wind Yuan <[email protected]>
diff --git a/capi/context_priv.cpp b/capi/context_priv.cpp
index 82085ac..a1a3b8f 100644
--- a/capi/context_priv.cpp
+++ b/capi/context_priv.cpp
@@ -210,9 +210,13 @@
return NULL;
}
+ CLStitchResMode res_mode = CLStitchRes1080P;
+ if (_res_mode == StitchRes4K)
+ res_mode = CLStitchRes4K;
+
SmartPtr<CLImage360Stitch> image_360 =
- create_image_360_stitch(
- context, _need_seam, _scale_mode, _fisheye_map).dynamic_cast_ptr<CLImage360Stitch> ();
+ create_image_360_stitch (context, _need_seam,
+ _scale_mode, _fisheye_map, res_mode).dynamic_cast_ptr<CLImage360Stitch> ();
XCAM_FAIL_RETURN (ERROR, image_360.ptr (), NULL, "create image stitch handler failed");
image_360->set_output_size (sttch_width, sttch_height);
XCAM_LOG_INFO ("stitch output size width:%d height:%d", sttch_width, sttch_height);
diff --git a/capi/context_priv.h b/capi/context_priv.h
index 5088f53..cbb67f2 100644
--- a/capi/context_priv.h
+++ b/capi/context_priv.h
@@ -159,12 +159,19 @@
: public ContextBase
{
public:
+ enum StitchResMode {
+ StitchRes1080P = 0,
+ StitchRes4K
+ };
+
+public:
StitchContext ()
: ContextBase (HandleTypeStitch)
, _need_seam (false)
, _fisheye_map (false)
, _fm_ocl (false)
, _scale_mode (CLBlenderScaleLocal)
+ , _res_mode (StitchRes1080P)
{}
virtual SmartPtr<CLImageHandler> create_handler (SmartPtr<CLContext> &context);
@@ -174,6 +181,7 @@
bool _fisheye_map;
bool _fm_ocl;
CLBlenderScaleMode _scale_mode;
+ StitchResMode _res_mode;
};
#endif //XCAM_CONTEXT_PRIV_H
diff --git a/cl_kernel/kernel_gauss_lap_pyramid.cl b/cl_kernel/kernel_gauss_lap_pyramid.cl
index 2457f42..f9067c0 100644
--- a/cl_kernel/kernel_gauss_lap_pyramid.cl
+++ b/cl_kernel/kernel_gauss_lap_pyramid.cl
@@ -246,12 +246,13 @@
#if !PYRAMID_UV
step_x = 0.125f / output_width;
- out_data = read_scale_y (input, sampler, normCoor, step_x) * 256.0f;
+ out_data = read_scale_y (input, sampler, normCoor, step_x) * 255.0f;
#else
step_x = 0.25f / output_width;
- out_data = read_scale_uv (input, sampler, normCoor, step_x) * 256.0f;
+ out_data = read_scale_uv (input, sampler, normCoor, step_x) * 255.0f;
#endif
+ out_data = clamp (out_data + 0.5f, 0.0f, 255.0f);
write_imageui (output, (int2)(g_x + out_offset_x, g_y), convert_uint4(as_ushort4(convert_uchar8(out_data))));
}
diff --git a/modules/ocl/cl_image_360_stitch.cpp b/modules/ocl/cl_image_360_stitch.cpp
index 0e8a33d..13206c2 100644
--- a/modules/ocl/cl_image_360_stitch.cpp
+++ b/modules/ocl/cl_image_360_stitch.cpp
@@ -100,50 +100,125 @@
return true;
}
+#if HAVE_OPENCV
+static CVFMConfig
+get_fm_default_config (CLStitchResMode res_mode)
+{
+ CVFMConfig config;
+
+ switch (res_mode) {
+ case CLStitchRes1080P: {
+ config.sitch_min_width = 56;
+ config.min_corners = 8;
+ config.offset_factor = 0.8f;
+ config.delta_mean_offset = 5.0f;
+ config.max_adjusted_offset = 12.0f;
+
+ break;
+ }
+ case CLStitchRes4K: {
+ config.sitch_min_width = 160;
+ config.min_corners = 8;
+ config.offset_factor = 0.8f;
+ config.delta_mean_offset = 5.0f;
+ config.max_adjusted_offset = 12.0f;
+
+ break;
+ }
+ default:
+ XCAM_LOG_DEBUG ("unknown reslution mode (%d)", res_mode);
+ break;
+ }
+
+ return config;
+}
+#endif
+
static CLStitchInfo
-get_default_stitch_info ()
+get_default_stitch_info (CLStitchResMode res_mode)
{
CLStitchInfo stitch_info;
- stitch_info.merge_width[0] = 56;
- stitch_info.merge_width[1] = 56;
+ switch (res_mode) {
+ case CLStitchRes1080P: {
+ stitch_info.merge_width[0] = 56;
+ stitch_info.merge_width[1] = 56;
- stitch_info.crop[0].left = 96;
- stitch_info.crop[0].right = 96;
- stitch_info.crop[0].top = 0;
- stitch_info.crop[0].bottom = 0;
- stitch_info.crop[1].left = 96;
- stitch_info.crop[1].right = 96;
- stitch_info.crop[1].top = 0;
- stitch_info.crop[1].bottom = 0;
+ stitch_info.crop[0].left = 96;
+ stitch_info.crop[0].right = 96;
+ stitch_info.crop[0].top = 0;
+ stitch_info.crop[0].bottom = 0;
+ stitch_info.crop[1].left = 96;
+ stitch_info.crop[1].right = 96;
+ stitch_info.crop[1].top = 0;
+ stitch_info.crop[1].bottom = 0;
- stitch_info.fisheye_info[0].center_x = 480.0f;
- stitch_info.fisheye_info[0].center_y = 480.0f;
- stitch_info.fisheye_info[0].wide_angle = 202.8f;
- stitch_info.fisheye_info[0].radius = 480.0f;
- stitch_info.fisheye_info[0].rotate_angle = -90.0f;
- stitch_info.fisheye_info[1].center_x = 1440.0f;
- stitch_info.fisheye_info[1].center_y = 480.0f;
- stitch_info.fisheye_info[1].wide_angle = 202.8f;
- stitch_info.fisheye_info[1].radius = 480.0f;
- stitch_info.fisheye_info[1].rotate_angle = 89.4f;
+ stitch_info.fisheye_info[0].center_x = 480.0f;
+ stitch_info.fisheye_info[0].center_y = 480.0f;
+ stitch_info.fisheye_info[0].wide_angle = 202.8f;
+ stitch_info.fisheye_info[0].radius = 480.0f;
+ stitch_info.fisheye_info[0].rotate_angle = -90.0f;
+ stitch_info.fisheye_info[1].center_x = 1440.0f;
+ stitch_info.fisheye_info[1].center_y = 480.0f;
+ stitch_info.fisheye_info[1].wide_angle = 202.8f;
+ stitch_info.fisheye_info[1].radius = 480.0f;
+ stitch_info.fisheye_info[1].rotate_angle = 89.4f;
+
+ break;
+ }
+ case CLStitchRes4K: {
+ stitch_info.merge_width[0] = 160;
+ stitch_info.merge_width[1] = 160;
+
+ stitch_info.crop[0].left = 64;
+ stitch_info.crop[0].right = 64;
+ stitch_info.crop[0].top = 0;
+ stitch_info.crop[0].bottom = 0;
+ stitch_info.crop[1].left = 64;
+ stitch_info.crop[1].right = 64;
+ stitch_info.crop[1].top = 0;
+ stitch_info.crop[1].bottom = 0;
+
+ stitch_info.fisheye_info[0].center_x = 1024.0f;
+ stitch_info.fisheye_info[0].center_y = 1024.0f;
+ stitch_info.fisheye_info[0].wide_angle = 195.0f;
+ stitch_info.fisheye_info[0].radius = 1040.0f;
+ stitch_info.fisheye_info[0].rotate_angle = 0.0f;
+
+ stitch_info.fisheye_info[1].center_x = 3072.0f;
+ stitch_info.fisheye_info[1].center_y = 1016.0f;
+ stitch_info.fisheye_info[1].wide_angle = 192.0f;
+ stitch_info.fisheye_info[1].radius = 1040.0f;
+ stitch_info.fisheye_info[1].rotate_angle = 0.4f;
+
+ break;
+ }
+ default:
+ XCAM_LOG_DEBUG ("unknown reslution mode (%d)", res_mode);
+ break;
+ }
return stitch_info;
}
-CLImage360Stitch::CLImage360Stitch (SmartPtr<CLContext> &context, CLBlenderScaleMode scale_mode)
+CLImage360Stitch::CLImage360Stitch (
+ SmartPtr<CLContext> &context, CLBlenderScaleMode scale_mode, CLStitchResMode res_mode)
: CLMultiImageHandler ("CLImage360Stitch")
, _context (context)
, _output_width (0)
, _output_height (0)
- , _is_stitch_inited (false)
, _scale_mode (scale_mode)
+ , _is_stitch_inited (false)
{
xcam_mem_clear (_merge_width);
#if HAVE_OPENCV
_feature_match = new CVFeatureMatch (context);
XCAM_ASSERT (_feature_match.ptr ());
+
+ _feature_match->set_config (get_fm_default_config (res_mode));
+#else
+ XCAM_UNUSED (res_mode);
#endif
}
@@ -530,8 +605,6 @@
CLImage360Stitch::prepare_parameters (SmartPtr<DrmBoBuffer> &input, SmartPtr<DrmBoBuffer> &output)
{
XCamReturn ret = XCAM_RETURN_NO_ERROR;
- if (!_is_stitch_inited)
- init_stitch_info (get_default_stitch_info ());
ret = prepare_fisheye_parameters (input, output);
STITCH_CHECK (ret, "prepare fisheye parameters failed");
@@ -670,15 +743,14 @@
}
SmartPtr<CLImageHandler>
-create_image_360_stitch (
- SmartPtr<CLContext> &context, bool need_seam,
- CLBlenderScaleMode scale_mode, bool fisheye_map)
+create_image_360_stitch (SmartPtr<CLContext> &context, bool need_seam,
+ CLBlenderScaleMode scale_mode, bool fisheye_map, CLStitchResMode res_mode)
{
const int layer = 2;
const bool need_uv = true;
SmartPtr<CLFisheyeHandler> fisheye;
SmartPtr<CLBlender> left_blender, right_blender;
- SmartPtr<CLImage360Stitch> stitch = new CLImage360Stitch (context, scale_mode);
+ SmartPtr<CLImage360Stitch> stitch = new CLImage360Stitch (context, scale_mode, res_mode);
XCAM_ASSERT (stitch.ptr ());
for (int index = 0; index < ImageIdxCount; ++index) {
@@ -709,6 +781,7 @@
}
}
+ stitch->init_stitch_info (get_default_stitch_info (res_mode));
return stitch;
}
diff --git a/modules/ocl/cl_image_360_stitch.h b/modules/ocl/cl_image_360_stitch.h
index 94dc118..bfbbcb4 100644
--- a/modules/ocl/cl_image_360_stitch.h
+++ b/modules/ocl/cl_image_360_stitch.h
@@ -31,6 +31,11 @@
namespace XCam {
+enum CLStitchResMode {
+ CLStitchRes1080P,
+ CLStitchRes4K
+};
+
enum ImageIdx {
ImageIdxMain,
ImageIdxSecondary,
@@ -94,7 +99,8 @@
: public CLMultiImageHandler
{
public:
- explicit CLImage360Stitch (SmartPtr<CLContext> &context, CLBlenderScaleMode scale_mode);
+ explicit CLImage360Stitch (
+ SmartPtr<CLContext> &context, CLBlenderScaleMode scale_mode, CLStitchResMode res_mode);
bool init_stitch_info (CLStitchInfo stitch_info);
void set_output_size (uint32_t width, uint32_t height) {
@@ -154,16 +160,18 @@
ImageMergeInfo _img_merge_info[ImageIdxCount];
Rect _overlaps[ImageIdxCount][2]; // 2=>Overlap0 and overlap1
- bool _is_stitch_inited;
-
CLBlenderScaleMode _scale_mode;
SmartPtr<BufferPool> _scale_buf_pool;
+
+ CLStitchResMode _res_mode;
+
+ bool _is_stitch_inited;
};
SmartPtr<CLImageHandler>
create_image_360_stitch (
- SmartPtr<CLContext> &context, bool need_seam = false,
- CLBlenderScaleMode scale_mode = CLBlenderScaleLocal, bool fisheye_map = false);
+ SmartPtr<CLContext> &context, bool need_seam = false, CLBlenderScaleMode scale_mode = CLBlenderScaleLocal,
+ bool fisheye_map = false, CLStitchResMode res_mode = CLStitchRes1080P);
}
diff --git a/modules/ocl/cl_post_image_processor.cpp b/modules/ocl/cl_post_image_processor.cpp
index aa7e425..001ee62 100644
--- a/modules/ocl/cl_post_image_processor.cpp
+++ b/modules/ocl/cl_post_image_processor.cpp
@@ -61,6 +61,7 @@
, _stitch_scale_mode (CLBlenderScaleLocal)
, _stitch_width (0)
, _stitch_height (0)
+ , _stitch_res_mode (0)
{
XCAM_LOG_DEBUG ("CLPostImageProcessor constructed");
}
@@ -386,9 +387,8 @@
/* image stitch */
image_handler =
- create_image_360_stitch (
- context, _stitch_enable_seam,
- _stitch_scale_mode, _stitch_fisheye_map);
+ create_image_360_stitch (context, _stitch_enable_seam,
+ _stitch_scale_mode, _stitch_fisheye_map, (CLStitchResMode) _stitch_res_mode);
_stitch = image_handler.dynamic_cast_ptr<CLImage360Stitch> ();
XCAM_FAIL_RETURN (
WARNING,
@@ -497,7 +497,7 @@
bool
CLPostImageProcessor::set_image_stitch (
bool enable_stitch, bool enable_seam, CLBlenderScaleMode scale_mode, bool enable_fisheye_map,
- bool fm_ocl, uint32_t stitch_width, uint32_t stitch_height)
+ bool fm_ocl, uint32_t stitch_width, uint32_t stitch_height, uint32_t res_mode)
{
XCAM_ASSERT (scale_mode < CLBlenderScaleMax);
@@ -511,6 +511,7 @@
_stitch_fisheye_map = enable_fisheye_map;
_stitch_width = stitch_width;
_stitch_height = stitch_height;
+ _stitch_res_mode = res_mode;
#if HAVE_OPENCV
_stitch_fm_ocl = fm_ocl;
diff --git a/modules/ocl/cl_post_image_processor.h b/modules/ocl/cl_post_image_processor.h
index 42f9e46..bb471e5 100644
--- a/modules/ocl/cl_post_image_processor.h
+++ b/modules/ocl/cl_post_image_processor.h
@@ -93,7 +93,7 @@
virtual bool set_image_warp (bool enable);
virtual bool set_image_stitch (
bool enable_stitch, bool enable_seam, CLBlenderScaleMode scale_mode, bool enable_fisheye_map,
- bool fm_ocl, uint32_t stitch_width, uint32_t stitch_height);
+ bool fm_ocl, uint32_t stitch_width, uint32_t stitch_height, uint32_t res_mode);
protected:
virtual bool can_process_result (SmartPtr<X3aResult> &result);
@@ -141,6 +141,7 @@
CLBlenderScaleMode _stitch_scale_mode;
uint32_t _stitch_width;
uint32_t _stitch_height;
+ uint32_t _stitch_res_mode;
};
};
diff --git a/modules/ocl/cv_feature_match.cpp b/modules/ocl/cv_feature_match.cpp
index b247270..5604f12 100644
--- a/modules/ocl/cv_feature_match.cpp
+++ b/modules/ocl/cv_feature_match.cpp
@@ -23,8 +23,34 @@
namespace XCam {
-#define XCAM_OF_DEBUG 0
-#define XCAM_OF_DRAW_SCALE 2
+#define XCAM_CV_FM_DEBUG 0
+#define XCAM_CV_OF_DRAW_SCALE 2
+
+#if XCAM_CV_FM_DEBUG
+static XCamReturn
+dump_buffer (SmartPtr<DrmBoBuffer> buffer, char *dump_name)
+{
+ ImageFileHandle file;
+
+ XCamReturn ret = file.open (dump_name, "wb");
+ if (ret != XCAM_RETURN_NO_ERROR) {
+ XCAM_LOG_ERROR ("open %s failed", dump_name);
+ return ret;
+ }
+
+ ret = file.write_buf (buffer);
+ if (ret != XCAM_RETURN_NO_ERROR) {
+ XCAM_LOG_ERROR ("write buffer to %s failed", dump_name);
+ file.close ();
+ return ret;
+ }
+
+ file.close ();
+ XCAM_LOG_INFO ("write buffer to %s done", dump_name);
+
+ return XCAM_RETURN_NO_ERROR;
+}
+#endif
CVFeatureMatch::CVFeatureMatch (SmartPtr<CLContext> &context)
: _context (context)
@@ -37,6 +63,12 @@
}
void
+CVFeatureMatch::set_config (const CVFMConfig config)
+{
+ _config = config;
+}
+
+void
CVFeatureMatch::init_opencv_ocl ()
{
if (_is_ocl_inited)
@@ -64,6 +96,31 @@
}
bool
+CVFeatureMatch::convert_to_mat (SmartPtr<CLContext> context, SmartPtr<DrmBoBuffer> buffer, cv::Mat &image)
+{
+ SmartPtr<CLBuffer> cl_buffer = new CLVaBuffer (context, buffer);
+ VideoBufferInfo info = buffer->get_video_info ();
+ cl_mem cl_mem_id = cl_buffer->get_mem_id ();
+
+ cv::UMat umat;
+ cv::ocl::convertFromBuffer (cl_mem_id, info.strides[0], info.height * 3 / 2, info.width, CV_8U, umat);
+ if (umat.empty ()) {
+ XCAM_LOG_ERROR ("convert bo buffer to UMat failed");
+ return false;
+ }
+
+ cv::Mat mat;
+ umat.copyTo (mat);
+ if (mat.empty ()) {
+ XCAM_LOG_ERROR ("copy UMat to Mat failed");
+ return false;
+ }
+
+ cv::cvtColor (mat, image, cv::COLOR_YUV2BGR_NV12);
+ return true;
+}
+
+bool
CVFeatureMatch::get_crop_image (
SmartPtr<DrmBoBuffer> buffer,
cv::Rect img_crop_left, cv::Rect img_crop_right,
@@ -108,9 +165,9 @@
count = 0;
sum = 0.0f;
for (uint32_t i = 0; i < status.size (); ++i) {
-#if XCAM_OF_DEBUG
- cv::Point start = cv::Point(corner0[i]) * XCAM_OF_DRAW_SCALE;
- cv::circle (image, start, 4, cv::Scalar(255, 255, 255), XCAM_OF_DRAW_SCALE);
+#if XCAM_CV_FM_DEBUG
+ cv::Point start = cv::Point(corner0[i]) * XCAM_CV_OF_DRAW_SCALE;
+ cv::circle (image, start, 4, cv::Scalar(255, 255, 255), XCAM_CV_OF_DRAW_SCALE);
#endif
if (!status[i] || error[i] > 24)
@@ -123,9 +180,9 @@
++count;
offsets.push_back (offset);
-#if XCAM_OF_DEBUG
- cv::Point end = (cv::Point(corner1[i]) + cv::Point (img0_size.width, 0)) * XCAM_OF_DRAW_SCALE;
- cv::line (image, start, end, cv::Scalar(255, 255, 255), XCAM_OF_DRAW_SCALE);
+#if XCAM_CV_FM_DEBUG
+ cv::Point end = (cv::Point(corner1[i]) + cv::Point (img0_size.width, 0)) * XCAM_CV_OF_DRAW_SCALE;
+ cv::line (image, start, end, cv::Scalar(255, 255, 255), XCAM_CV_OF_DRAW_SCALE);
#else
XCAM_UNUSED (image);
XCAM_UNUSED (img0_size);
@@ -141,7 +198,7 @@
mean_offset = sum / count;
-#if XCAM_OF_DEBUG
+#if XCAM_CV_FM_DEBUG
XCAM_LOG_INFO (
"X-axis mean offset:%.2f, pre_mean_offset:%.2f (%d times, count:%d)",
mean_offset, 0.0f, 0, count);
@@ -167,7 +224,7 @@
}
mean_offset = sum / recur_count;
-#if XCAM_OF_DEBUG
+#if XCAM_CV_FM_DEBUG
XCAM_LOG_INFO (
"X-axis mean offset:%.2f, pre_mean_offset:%.2f (%d times, count:%d)",
mean_offset, pre_mean_offset, try_times, recur_count);
@@ -201,7 +258,7 @@
cv::Size img1_size = image1.size ();
XCAM_ASSERT (img0_size.height == img1_size.height);
-#if XCAM_OF_DEBUG
+#if XCAM_CV_FM_DEBUG
cv::Mat mat;
cv::UMat umat;
cv::Size size (img0_size.width + img1_size.width, img0_size.height);
@@ -222,7 +279,7 @@
mat.copyTo (out_image);
}
- cv::Size scale_size = size * XCAM_OF_DRAW_SCALE;
+ cv::Size scale_size = size * XCAM_CV_OF_DRAW_SCALE;
cv::resize (out_image, out_image, scale_size, 0, 0);
#endif
@@ -247,9 +304,9 @@
last_count = count;
last_mean_offset = mean_offset;
-#if XCAM_OF_DEBUG
+#if XCAM_CV_FM_DEBUG
char file_name[1024];
- snprintf (file_name, 1023, "feature_match_%d_OF_%d.jpg", frame_num, idx);
+ snprintf (file_name, 1023, "fm_optical_flow_%d_%d.jpg", frame_num, idx);
cv::imwrite (file_name, out_image);
XCAM_LOG_INFO ("write feature match: %s", file_name);
#else
@@ -301,7 +358,7 @@
add_detected_data (img_left, fast_detector, corner_left);
if (corner_left.empty ()) {
-#if XCAM_OF_DEBUG
+#if XCAM_CV_FM_DEBUG
if (idx == 1)
frame_num++;
idx = (idx == 0) ? 1 : 0;
@@ -316,7 +373,7 @@
status, err, valid_count, mean_offset, x_offset, frame_num, idx);
adjust_stitch_area (dst_width, x_offset, crop_left, crop_right);
-#if XCAM_OF_DEBUG
+#if XCAM_CV_FM_DEBUG
XCAM_LOG_INFO (
"Stiching area %d: left_area(x:%d, width:%d), right_area(x:%d, width:%d)",
idx, crop_left.x, crop_left.width, crop_right.x, crop_right.width);
@@ -362,6 +419,40 @@
_valid_count[0], _mean_offset[0], _x_offset[0], dst_width);
detect_and_match (img0_right, img1_left, img0_crop_right, img1_crop_left,
_valid_count[1], _mean_offset[1], _x_offset[1], dst_width);
+
+#if XCAM_CV_FM_DEBUG
+ static int frame = 0;
+ char file_name[1024];
+ VideoBufferInfo info = buf0->get_video_info ();
+
+ std::snprintf (file_name, 1023, "fm_in_%dx%d_%d_0.nv12", info.width, info.height, frame);
+ dump_buffer (buf0, file_name);
+ std::snprintf (file_name, 1023, "fm_in_%dx%d_%d_1.nv12", info.width, info.height, frame);
+ dump_buffer (buf1, file_name);
+
+ cv::Mat in_mat;
+ std::snprintf (file_name, 1023, "fm_in_stitch_area_%d_0.jpg", frame);
+ convert_to_mat (_context, buf0, in_mat);
+ cv::line (in_mat, cv::Point(img0_crop_left.x, 0), cv::Point(img0_crop_left.x, dst_width), cv::Scalar(0, 0, 255), 2);
+ cv::line (in_mat, cv::Point(img0_crop_left.x + img0_crop_left.width, 0),
+ cv::Point(img0_crop_left.x + img0_crop_left.width, dst_width), cv::Scalar(0, 0, 255), 2);
+ cv::line (in_mat, cv::Point(img0_crop_right.x, 0), cv::Point(img0_crop_right.x, dst_width), cv::Scalar(0, 0, 255), 2);
+ cv::line (in_mat, cv::Point(img0_crop_right.x + img0_crop_right.width, 0),
+ cv::Point(img0_crop_right.x + img0_crop_right.width, dst_width), cv::Scalar(0, 0, 255), 2);
+ cv::imwrite (file_name, in_mat);
+
+ std::snprintf (file_name, 1023, "fm_in_stitch_area_%d_1.jpg", frame);
+ convert_to_mat (_context, buf1, in_mat);
+ cv::line (in_mat, cv::Point(img1_crop_left.x, 0), cv::Point(img1_crop_left.x, dst_width), cv::Scalar(0, 0, 255), 2);
+ cv::line (in_mat, cv::Point(img1_crop_left.x + img1_crop_left.width, 0),
+ cv::Point(img1_crop_left.x + img1_crop_left.width, dst_width), cv::Scalar(0, 0, 255), 2);
+ cv::line (in_mat, cv::Point(img1_crop_right.x, 0), cv::Point(img1_crop_right.x, dst_width), cv::Scalar(0, 0, 255), 2);
+ cv::line (in_mat, cv::Point(img1_crop_right.x + img1_crop_right.width, 0),
+ cv::Point(img1_crop_right.x + img1_crop_right.width, dst_width), cv::Scalar(0, 0, 255), 2);
+ cv::imwrite (file_name, in_mat);
+
+ frame++;
+#endif
}
}
diff --git a/modules/ocl/cv_feature_match.h b/modules/ocl/cv_feature_match.h
index 68c1cc5..1c9645f 100644
--- a/modules/ocl/cv_feature_match.h
+++ b/modules/ocl/cv_feature_match.h
@@ -27,6 +27,7 @@
#include <dma_video_buffer.h>
#include <smartptr.h>
#include "xcam_obj_debug.h"
+#include "image_file_handle.h"
#include <cl_context.h>
#include <cl_device.h>
@@ -67,6 +68,8 @@
return _use_ocl;
}
+ void set_config (const CVFMConfig config);
+
void optical_flow_feature_match (
int output_width, SmartPtr<DrmBoBuffer> buf0, SmartPtr<DrmBoBuffer> buf1,
cv::Rect &img0_crop_left, cv::Rect &img0_crop_right, cv::Rect &img1_crop_left, cv::Rect &img1_crop_right);
@@ -74,6 +77,7 @@
protected:
void init_opencv_ocl ();
+ bool convert_to_mat (SmartPtr<CLContext> context, SmartPtr<DrmBoBuffer> buffer, cv::Mat &image);
bool get_crop_image (SmartPtr<DrmBoBuffer> buffer,
cv::Rect img_crop_left, cv::Rect img_crop_right, cv::UMat &img_left, cv::UMat &img_right);
diff --git a/tests/test-image-stitching.cpp b/tests/test-image-stitching.cpp
index 83736bf..512f2c4 100644
--- a/tests/test-image-stitching.cpp
+++ b/tests/test-image-stitching.cpp
@@ -30,42 +30,11 @@
#include "cl_fisheye_handler.h"
#include "cl_image_360_stitch.h"
-#define XCAM_STITCHING_DEBUG 0
+#define XCAM_TEST_STITCH_DEBUG 0
#define XCAM_ALIGNED_WIDTH 16
using namespace XCam;
-static CLStitchInfo
-get_stitch_initial_info ()
-{
- CLStitchInfo stitch_info;
-
- stitch_info.merge_width[0] = 56;
- stitch_info.merge_width[1] = 56;
-
- stitch_info.crop[0].left = 96;
- stitch_info.crop[0].right = 96;
- stitch_info.crop[0].top = 0;
- stitch_info.crop[0].bottom = 0;
- stitch_info.crop[1].left = 96;
- stitch_info.crop[1].right = 96;
- stitch_info.crop[1].top = 0;
- stitch_info.crop[1].bottom = 0;
-
- stitch_info.fisheye_info[0].center_x = 480.0f;
- stitch_info.fisheye_info[0].center_y = 480.0f;
- stitch_info.fisheye_info[0].wide_angle = 202.8f;
- stitch_info.fisheye_info[0].radius = 480.0f;
- stitch_info.fisheye_info[0].rotate_angle = -90.0f;
- stitch_info.fisheye_info[1].center_x = 1440.0f;
- stitch_info.fisheye_info[1].center_y = 480.0f;
- stitch_info.fisheye_info[1].wide_angle = 202.8f;
- stitch_info.fisheye_info[1].radius = 480.0f;
- stitch_info.fisheye_info[1].rotate_angle = 89.4f;
-
- return stitch_info;
-}
-
#if HAVE_OPENCV
void
init_opencv_ocl (SmartPtr<CLContext> context)
@@ -113,14 +82,16 @@
"\t--input-h optional, input height, default: 1080\n"
"\t--output-w optional, output width, default: 1920\n"
"\t--output-h optional, output width, default: 960\n"
- "\t--loop optional, how many loops need to run for performance test, default: 0\n"
- "\t--save optional, save file or not, select from [true/false], default: true\n"
+ "\t--res-mode optional, image resolution mode, select from [1080p/4k], default: 1080p\n"
"\t--scale-mode optional, image scaling mode, select from [local/global], default: local\n"
"\t--enable-seam optional, enable seam finder in blending area, default: no\n"
"\t--enable-fisheyemap optional, enable fisheye map, default: no\n"
#if HAVE_OPENCV
"\t--fm-ocl optional, enable ocl for feature match, select from [true/false], default: false\n"
#endif
+ "\t--save optional, save file or not, select from [true/false], default: true\n"
+ "\t--framerate optional, framerate of saved video, default: 30.0\n"
+ "\t--loop optional, how many loops need to run for performance test, default: 1\n"
"\t--help usage\n",
arg0);
}
@@ -168,11 +139,15 @@
int loop = 1;
bool enable_seam = false;
bool enable_fisheye_map = false;
+ CLBlenderScaleMode scale_mode = CLBlenderScaleLocal;
+ CLStitchResMode res_mode = CLStitchRes1080P;
+
#if HAVE_OPENCV
bool fm_ocl = false;
#endif
bool need_save_output = true;
- CLBlenderScaleMode scale_mode = CLBlenderScaleLocal;
+ double framerate = 30.0;
+
const char *file_in_name = NULL;
const char *file_out_name = NULL;
@@ -183,14 +158,16 @@
{"input-h", required_argument, NULL, 'h'},
{"output-w", required_argument, NULL, 'W'},
{"output-h", required_argument, NULL, 'H'},
- {"loop", required_argument, NULL, 'l'},
- {"save", required_argument, NULL, 's'},
+ {"res-mode", required_argument, NULL, 'R'},
{"scale-mode", required_argument, NULL, 'c'},
{"enable-seam", no_argument, NULL, 'S'},
{"enable-fisheyemap", no_argument, NULL, 'F'},
#if HAVE_OPENCV
{"fm-ocl", required_argument, NULL, 'O'},
#endif
+ {"save", required_argument, NULL, 's'},
+ {"framerate", required_argument, NULL, 'f'},
+ {"loop", required_argument, NULL, 'l'},
{"help", no_argument, NULL, 'e'},
{NULL, 0, NULL, 0},
};
@@ -218,11 +195,15 @@
case 'H':
output_height = atoi(optarg);
break;
- case 'l':
- loop = atoi(optarg);
- break;
- case 's':
- need_save_output = (strcasecmp (optarg, "false") == 0 ? false : true);
+ case 'R':
+ if (!strcasecmp (optarg, "1080p"))
+ res_mode = CLStitchRes1080P;
+ else if (!strcasecmp (optarg, "4k"))
+ res_mode = CLStitchRes4K;
+ else {
+ XCAM_LOG_ERROR ("incorrect resolution mode");
+ return -1;
+ }
break;
case 'c':
if (!strcasecmp (optarg, "local"))
@@ -245,6 +226,15 @@
fm_ocl = (strcasecmp (optarg, "true") == 0 ? true : false);
break;
#endif
+ case 's':
+ need_save_output = (strcasecmp (optarg, "false") == 0 ? false : true);
+ break;
+ case 'f':
+ framerate = atof(optarg);
+ break;
+ case 'l':
+ loop = atoi(optarg);
+ break;
case 'e':
usage (argv[0]);
return -1;
@@ -273,6 +263,13 @@
return -1;
}
+#if !HAVE_OPENCV
+ if (need_save_output) {
+ XCAM_LOG_WARNING ("non-OpenCV mode, can't save video");
+ need_save_output = false;
+ }
+#endif
+
printf ("Description------------------------\n");
printf ("input file:\t\t%s\n", file_in_name);
printf ("output file:\t\t%s\n", file_out_name);
@@ -280,24 +277,24 @@
printf ("input height:\t\t%d\n", input_height);
printf ("output width:\t\t%d\n", output_width);
printf ("output height:\t\t%d\n", output_height);
- printf ("loop count:\t\t%d\n", loop);
- printf ("save file:\t\t%s\n", need_save_output ? "true" : "false");
+ printf ("resolution mode:\t%s\n", res_mode == CLStitchRes1080P ? "1080P" : "4K");
printf ("scale mode:\t\t%s\n", scale_mode == CLBlenderScaleLocal ? "local" : "global");
printf ("seam mask:\t\t%s\n", enable_seam ? "true" : "false");
printf ("fisheye map:\t\t%s\n", enable_fisheye_map ? "true" : "false");
#if HAVE_OPENCV
printf ("feature match ocl:\t%s\n", fm_ocl ? "true" : "false");
#endif
+ printf ("save file:\t\t%s\n", need_save_output ? "true" : "false");
+ printf ("framerate:\t\t%.3lf\n", framerate);
+ printf ("loop count:\t\t%d\n", loop);
printf ("-----------------------------------\n");
context = CLDevice::instance ()->get_context ();
image_360 =
- create_image_360_stitch (
- context, enable_seam, scale_mode, enable_fisheye_map).dynamic_cast_ptr<CLImage360Stitch> ();
+ create_image_360_stitch (context, enable_seam,
+ scale_mode, enable_fisheye_map, res_mode).dynamic_cast_ptr<CLImage360Stitch> ();
XCAM_ASSERT (image_360.ptr ());
image_360->set_output_size (output_width, output_height);
- CLStitchInfo stitch_info = get_stitch_initial_info ();
- image_360->init_stitch_info (stitch_info);
#if HAVE_OPENCV
image_360->set_feature_match_ocl (fm_ocl);
#endif
@@ -323,7 +320,7 @@
cv::VideoWriter writer;
if (need_save_output) {
cv::Size dst_size = cv::Size (output_width, output_height);
- if (!writer.open (file_out_name, CV_FOURCC('X', '2', '6', '4'), 30, dst_size)) {
+ if (!writer.open (file_out_name, CV_FOURCC('X', '2', '6', '4'), framerate, dst_size)) {
XCAM_LOG_ERROR ("open file %s failed", file_out_name);
return -1;
}
@@ -350,15 +347,36 @@
ret = image_360->execute (input_buf, output_buf);
CHECK (ret, "image_360 stitch execute failed");
- if (need_save_output) {
#if HAVE_OPENCV
+ if (need_save_output) {
cv::Mat out_mat;
convert_to_mat (context, output_buf, out_mat);
- writer.write (out_mat);
+#if XCAM_TEST_STITCH_DEBUG
+ static int frame = 0;
+ char file_name [1024];
+ std::snprintf (file_name, 1023, "orig_fisheye_%d.jpg", frame);
+
+ cv::Mat in_mat;
+ convert_to_mat (context, input_buf, in_mat);
+ cv::circle (in_mat, cv::Point(stitch_info.fisheye_info[0].center_x, stitch_info.fisheye_info[0].center_y),
+ stitch_info.fisheye_info[0].radius, cv::Scalar(0, 0, 255), 2);
+ cv::circle (in_mat, cv::Point(stitch_info.fisheye_info[1].center_x, stitch_info.fisheye_info[1].center_y),
+ stitch_info.fisheye_info[1].radius, cv::Scalar(0, 255, 0), 2);
+ cv::imwrite (file_name, in_mat);
+
+ char frame_str[1024];
+ std::snprintf (frame_str, 1023, "%d", frame);
+ cv::putText (out_mat, frame_str, cv::Point(120, 120), cv::FONT_HERSHEY_COMPLEX, 2.0,
+ cv::Scalar(0, 0, 255), 2, 8, false);
+ std::snprintf (file_name, 1023, "stitched_img_%d.jpg", frame);
+ cv::imwrite (file_name, out_mat);
+
+ frame++;
#endif
- } else {
+ writer.write (out_mat);
+ } else
+#endif
ensure_gpu_buffer_done (output_buf);
- }
FPS_CALCULATION (image_stitching, XCAM_OBJ_DUR_FRAME_NUM);
++i;
diff --git a/wrapper/gstreamer/gstxcamfilter.cpp b/wrapper/gstreamer/gstxcamfilter.cpp
index 0e686d3..fdbef97 100644
--- a/wrapper/gstreamer/gstxcamfilter.cpp
+++ b/wrapper/gstreamer/gstxcamfilter.cpp
@@ -42,6 +42,7 @@
#define DEFAULT_PROP_STITCH_SCALE_MODE CLBlenderScaleLocal
#define DEFAULT_PROP_STITCH_FISHEYE_MAP FALSE
#define DEFAULT_PROP_STITCH_FM_OCL FALSE
+#define DEFAULT_PROP_STITCH_RES_MODE StitchRes1080P
XCAM_BEGIN_DECLARE
@@ -58,7 +59,8 @@
PROP_STITCH_ENABLE_SEAM,
PROP_STITCH_SCALE_MODE,
PROP_STITCH_FISHEYE_MAP,
- PROP_STITCH_FM_OCL
+ PROP_STITCH_FM_OCL,
+ PROP_STITCH_RES_MODE
};
#define GST_TYPE_XCAM_FILTER_COPY_MODE (gst_xcam_filter_copy_mode_get_type ())
@@ -168,6 +170,26 @@
return g_type;
}
+#define GST_TYPE_XCAM_FILTER_STITCH_RES_MODE (gst_xcam_filter_stitch_res_mode_get_type ())
+static GType
+gst_xcam_filter_stitch_res_mode_get_type (void)
+{
+ static GType g_type = 0;
+ static const GEnumValue stitch_res_mode_types [] = {
+ {StitchRes1080P, "Image stitch 1080P mode", "1080p"},
+ {StitchRes4K, "Image stitch 4K mode", "4k"},
+ {0, NULL, NULL}
+ };
+
+ if (g_once_init_enter (&g_type)) {
+ const GType type =
+ g_enum_register_static ("GstXCamFilterStitchResModeType", stitch_res_mode_types);
+ g_once_init_leave (&g_type, type);
+ }
+
+ return g_type;
+}
+
static GstStaticPadTemplate gst_xcam_sink_factory =
GST_STATIC_PAD_TEMPLATE ("sink",
GST_PAD_SINK,
@@ -285,6 +307,12 @@
DEFAULT_PROP_STITCH_FM_OCL, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
#endif
+ g_object_class_install_property (
+ gobject_class, PROP_STITCH_RES_MODE,
+ g_param_spec_enum ("stitch-res-mode", "stitch resolution mode", "Stitch Resolution Mode",
+ GST_TYPE_XCAM_FILTER_STITCH_RES_MODE, DEFAULT_PROP_STITCH_RES_MODE,
+ (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+
gst_element_class_set_details_simple (element_class,
"Libxcam Filter",
"Filter/Effect/Video",
@@ -321,6 +349,7 @@
xcamfilter->stitch_fisheye_map = DEFAULT_PROP_STITCH_FISHEYE_MAP;
xcamfilter->stitch_fm_ocl = DEFAULT_PROP_STITCH_FM_OCL;
xcamfilter->stitch_scale_mode = DEFAULT_PROP_STITCH_SCALE_MODE;
+ xcamfilter->stitch_res_mode = DEFAULT_PROP_STITCH_RES_MODE;
xcamfilter->delay_buf_num = DEFAULT_DELAY_BUFFER_NUM;
xcamfilter->cached_buf_num = 0;
@@ -388,6 +417,9 @@
xcamfilter->stitch_fm_ocl = g_value_get_boolean (value);
break;
#endif
+ case PROP_STITCH_RES_MODE:
+ xcamfilter->stitch_res_mode = (StitchResMode) g_value_get_enum (value);
+ break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
@@ -438,6 +470,9 @@
g_value_set_boolean (value, xcamfilter->stitch_fm_ocl);
break;
#endif
+ case PROP_STITCH_RES_MODE:
+ g_value_set_enum (value, xcamfilter->stitch_res_mode);
+ break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
@@ -684,7 +719,7 @@
processor->set_image_stitch (
xcamfilter->enable_stitch, xcamfilter->stitch_enable_seam,
xcamfilter->stitch_scale_mode, xcamfilter->stitch_fisheye_map, xcamfilter->stitch_fm_ocl,
- GST_VIDEO_INFO_WIDTH (&out_info), GST_VIDEO_INFO_HEIGHT (&out_info));
+ GST_VIDEO_INFO_WIDTH (&out_info), GST_VIDEO_INFO_HEIGHT (&out_info), (uint32_t) xcamfilter->stitch_res_mode);
XCAM_LOG_INFO ("xcamfilter stitch output size width:%d height:%d",
GST_VIDEO_INFO_WIDTH (&out_info), GST_VIDEO_INFO_HEIGHT (&out_info));
}
diff --git a/wrapper/gstreamer/gstxcamfilter.h b/wrapper/gstreamer/gstxcamfilter.h
index a41b22f..b7ccd5b 100644
--- a/wrapper/gstreamer/gstxcamfilter.h
+++ b/wrapper/gstreamer/gstxcamfilter.h
@@ -67,6 +67,11 @@
DENOISE_3D_UV
} Denoise3DModeType;
+enum StitchResMode {
+ StitchRes1080P = 0,
+ StitchRes4K
+};
+
typedef struct _GstXCamFilter GstXCamFilter;
typedef struct _GstXCamFilterClass GstXCamFilterClass;
@@ -87,6 +92,7 @@
gboolean stitch_fisheye_map;
gboolean stitch_fm_ocl;
CLBlenderScaleMode stitch_scale_mode;
+ StitchResMode stitch_res_mode;
uint32_t delay_buf_num;
uint32_t cached_buf_num;
