plugins/smart/dvs/libdvs/stabilizer.cpp

/*
 * stablizer.cpp - abstract for DVS (Digital Video Stabilizer)
 *
 *    Copyright (c) 2014-2016 Intel Corporation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Author: Zong Wei <wei.zong@intel.com>
 */

#include "stabilizer.h"

using namespace cv;
using namespace cv::videostab;
using namespace std;

Mat
OnePassVideoStabilizer::nextStabilizedMotion(DvsData* frame, int& stablizedPos)
{
    if (!(frame->data.empty()))
    {
        Mat image;
        frame->data.getMat(ACCESS_READ).copyTo(image);

        curPos_++;

        if (curPos_ > 0)
        {
            at(curPos_, frames_) = image;

            if (doDeblurring_)
                at(curPos_, blurrinessRates_) = calcBlurriness(image);

            at(curPos_ - 1, motions_) = estimateMotion();

            if (curPos_ >= radius_)
            {
                curStabilizedPos_ = curPos_ - radius_;
                Mat stabilizationMotion = estimateStabilizationMotion();
                if (doCorrectionForInclusion_)
                    stabilizationMotion = ensureInclusionConstraint(stabilizationMotion, frameSize_, trimRatio_);

                at(curStabilizedPos_, stabilizationMotions_) = stabilizationMotion;
                stablizedPos = curStabilizedPos_;

                return stabilizationMotion;
            }
        }
        else
            setUpFrame(image);

        log_->print(".");
        return Mat();
    }

    if (curStabilizedPos_ < curPos_)
    {
        curStabilizedPos_++;
        stablizedPos = curStabilizedPos_;
        at(curStabilizedPos_ + radius_, frames_) = at(curPos_, frames_);
        at(curStabilizedPos_ + radius_ - 1, motions_) = Mat::eye(3, 3, CV_32F);

        Mat stabilizationMotion = estimateStabilizationMotion();
        if (doCorrectionForInclusion_)
            stabilizationMotion = ensureInclusionConstraint(stabilizationMotion, frameSize_, trimRatio_);

        at(curStabilizedPos_, stabilizationMotions_) = stabilizationMotion;

        log_->print(".");

        return stabilizationMotion;
    }

    return Mat();
}


Mat
OnePassVideoStabilizer::estimateMotion()
{
#if ENABLE_DVS_CL_PATH
    cv::UMat frame0 = at(curPos_ - 1, frames_).getUMat(ACCESS_READ);
    cv::UMat frame1 = at(curPos_, frames_).getUMat(ACCESS_READ);

    cv::UMat ugrayImage0;
    cv::UMat ugrayImage1;
    if ( frame0.type() != CV_8U )
    {
        cvtColor( frame0, ugrayImage0, COLOR_BGR2GRAY );
    }
    else
    {
        ugrayImage0 = frame0;
    }

    if ( frame1.type() != CV_8U )
    {
        cvtColor( frame1, ugrayImage1, COLOR_BGR2GRAY );
    }
    else
    {
        ugrayImage1 = frame1;
    }

    return motionEstimator_.dynamicCast<KeypointBasedMotionEstimator>()->estimate(ugrayImage0, ugrayImage1);
#else
    return motionEstimator_.dynamicCast<KeypointBasedMotionEstimator>()->estimate(at(curPos_ - 1, frames_), at(curPos_, frames_));
#endif
}

void
OnePassVideoStabilizer::setUpFrame(const Mat &firstFrame)
{
    frameSize_ = firstFrame.size();
    frameMask_.create(frameSize_, CV_8U);
    frameMask_.setTo(255);

    int cacheSize = 2 * radius_ + 1;
    frames_.resize(2);
    motions_.resize(cacheSize);
    stabilizationMotions_.resize(cacheSize);

    for (int i = -radius_; i < 0; ++i)
    {
        at(i, motions_) = Mat::eye(3, 3, CV_32F);
        at(i, frames_) = firstFrame;
    }

    at(0, frames_) = firstFrame;

    StabilizerBase::setUp(firstFrame);
}


Mat
TwoPassVideoStabilizer::nextStabilizedMotion(DvsData* frame, int& stablizedPos)
{
    if (!(frame->data.empty()))
    {
        Mat image;
        frame->data.getMat(ACCESS_READ).copyTo(image);

        curPos_++;

        if (curPos_ > 0)
        {
            at(curPos_, frames_) = image;

            if (doDeblurring_)
                at(curPos_, blurrinessRates_) = calcBlurriness(image);

            at(curPos_ - 1, motions_) = estimateMotion();

            if (curPos_ >= radius_)
            {
                curStabilizedPos_ = curPos_ - radius_;
                Mat stabilizationMotion = estimateStabilizationMotion();
                if (doCorrectionForInclusion_)
                    stabilizationMotion = ensureInclusionConstraint(stabilizationMotion, frameSize_, trimRatio_);

                at(curStabilizedPos_, stabilizationMotions_) = stabilizationMotion;
                stablizedPos = curStabilizedPos_;

                return stabilizationMotion;
            }
        }
        else
            setUpFrame(image);

        log_->print(".");
        return Mat();
    }

    if (curStabilizedPos_ < curPos_)
    {
        curStabilizedPos_++;
        stablizedPos = curStabilizedPos_;
        at(curStabilizedPos_ + radius_, frames_) = at(curPos_, frames_);
        at(curStabilizedPos_ + radius_ - 1, motions_) = Mat::eye(3, 3, CV_32F);

        Mat stabilizationMotion = estimateStabilizationMotion();
        if (doCorrectionForInclusion_)
            stabilizationMotion = ensureInclusionConstraint(stabilizationMotion, frameSize_, trimRatio_);

        at(curStabilizedPos_, stabilizationMotions_) = stabilizationMotion;

        log_->print(".");

        return stabilizationMotion;
    }

    return Mat();
}


Mat
TwoPassVideoStabilizer::estimateMotion()
{
#if ENABLE_DVS_CL_PATH
    cv::UMat frame0 = at(curPos_ - 1, frames_).getUMat(ACCESS_READ);
    cv::UMat frame1 = at(curPos_, frames_).getUMat(ACCESS_READ);

    cv::UMat ugrayImage0;
    cv::UMat ugrayImage1;
    if ( frame0.type() != CV_8U )
    {
        cvtColor( frame0, ugrayImage0, COLOR_BGR2GRAY );
    }
    else
    {
        ugrayImage0 = frame0;
    }

    if ( frame1.type() != CV_8U )
    {
        cvtColor( frame1, ugrayImage1, COLOR_BGR2GRAY );
    }
    else
    {
        ugrayImage1 = frame1;
    }

    return motionEstimator_.dynamicCast<KeypointBasedMotionEstimator>()->estimate(ugrayImage0, ugrayImage1);
#else
    return motionEstimator_.dynamicCast<KeypointBasedMotionEstimator>()->estimate(at(curPos_ - 1, frames_), at(curPos_, frames_));
#endif
}

void
TwoPassVideoStabilizer::setUpFrame(const Mat &firstFrame)
{
    //int cacheSize = 2*radius_ + 1;
    frames_.resize(2);
    stabilizedFrames_.resize(2);
    stabilizedMasks_.resize(2);

    for (int i = -1; i <= 0; ++i)
        at(i, frames_) = firstFrame;

    StabilizerBase::setUp(firstFrame);
}

VideoStabilizer::VideoStabilizer(
    bool isTwoPass,
    bool wobbleSuppress,
    bool deblur,
    bool inpainter)
    : isTwoPass_ (isTwoPass)
    , trimRatio_ (0.05f)
{
    Ptr<MotionEstimatorRansacL2> est = makePtr<MotionEstimatorRansacL2>(MM_HOMOGRAPHY);
    Ptr<IOutlierRejector> outlierRejector = makePtr<TranslationBasedLocalOutlierRejector>();
    Ptr<KeypointBasedMotionEstimator> kbest = makePtr<KeypointBasedMotionEstimator>(est);
    kbest->setDetector(GFTTDetector::create(1000, 0.01, 15));
    kbest->setOutlierRejector(outlierRejector);

    if (isTwoPass)
    {
        Ptr<TwoPassVideoStabilizer> twoPassStabilizer = makePtr<TwoPassVideoStabilizer>();
        stabilizer_ = twoPassStabilizer;
        twoPassStabilizer->setEstimateTrimRatio(false);
        twoPassStabilizer->setMotionStabilizer(makePtr<GaussianMotionFilter>(15, 10));

        if (wobbleSuppress) {
            Ptr<MoreAccurateMotionWobbleSuppressorBase> ws = makePtr<MoreAccurateMotionWobbleSuppressor>();

            ws->setMotionEstimator(kbest);
            ws->setPeriod(30);
            twoPassStabilizer->setWobbleSuppressor(ws);
        }
    } else {
        Ptr<OnePassVideoStabilizer> onePassStabilizer = makePtr<OnePassVideoStabilizer>();
        stabilizer_ = onePassStabilizer;
        onePassStabilizer->setMotionFilter(makePtr<GaussianMotionFilter>(15, 10));
    }

    stabilizer_->setMotionEstimator(kbest);

    stabilizer_->setRadius(15);

    if (deblur)
    {
        Ptr<WeightingDeblurer> deblurrer = makePtr<WeightingDeblurer>();
        deblurrer->setRadius(3);
        deblurrer->setSensitivity(0.001f);
        stabilizer_->setDeblurer(deblurrer);
    }

    if (inpainter)
    {
        bool inpaintMosaic = true;
        bool inpaintColorAverage = true;
        bool inpaintColorNs = false;
        bool inpaintColorTelea = false;

        // init inpainter
        InpaintingPipeline *inpainters = new InpaintingPipeline();
        Ptr<InpainterBase> inpainters_(inpainters);
        if (true == inpaintMosaic)
        {
            Ptr<ConsistentMosaicInpainter> inp = makePtr<ConsistentMosaicInpainter>();
            inp->setStdevThresh(10.0f);
            inpainters->pushBack(inp);
        }
        if (true == inpaintColorAverage)
            inpainters->pushBack(makePtr<ColorAverageInpainter>());
        else if (true == inpaintColorNs)
            inpainters->pushBack(makePtr<ColorInpainter>(0, 2));
        else if (true == inpaintColorTelea)
            inpainters->pushBack(makePtr<ColorInpainter>(1, 2));
        if (!inpainters->empty())
        {
            inpainters->setRadius(2);
            stabilizer_->setInpainter(inpainters_);
        }
    }
}

VideoStabilizer::~VideoStabilizer() {}

void
VideoStabilizer::configFeatureDetector(int features, double minDistance)
{
    Ptr<ImageMotionEstimatorBase> estimator = stabilizer_->motionEstimator();
    Ptr<FeatureDetector> detector = estimator.dynamicCast<KeypointBasedMotionEstimator>()->detector();
    if (NULL == detector) {
        return;
    }

    detector.dynamicCast<GFTTDetector>()->setMaxFeatures(features);
    detector.dynamicCast<GFTTDetector>()->setMinDistance(minDistance);
}

void
VideoStabilizer::configMotionFilter(int radius, float stdev)
{
    if (isTwoPass_) {
        Ptr<TwoPassVideoStabilizer> stab = stabilizer_.dynamicCast<TwoPassVideoStabilizer>();
        Ptr<IMotionStabilizer> motionStabilizer = stab->motionStabilizer();
        motionStabilizer.dynamicCast<GaussianMotionFilter>()->setParams(radius, stdev);
    } else {
        Ptr<OnePassVideoStabilizer> stab = stabilizer_.dynamicCast<OnePassVideoStabilizer>();
        Ptr<MotionFilterBase> motionFilter = stab->motionFilter();
        motionFilter.dynamicCast<GaussianMotionFilter>()->setParams(radius, stdev);
    }
    stabilizer_->setRadius(radius);
}

void
VideoStabilizer::configDeblurrer(int radius, double sensitivity)
{
    Ptr<DeblurerBase> deblurrer = stabilizer_->deblurrer();
    if (NULL == deblurrer) {
        return;
    }

    deblurrer->setRadius(radius);
    deblurrer.dynamicCast<WeightingDeblurer>()->setSensitivity(sensitivity);
}

void
VideoStabilizer::setFrameSize(Size frameSize)
{
    frameSize_ = frameSize;
}

Mat
VideoStabilizer::nextFrame()
{
    if (isTwoPass_) {
        Ptr<TwoPassVideoStabilizer> stab = stabilizer_.dynamicCast<TwoPassVideoStabilizer>();
        return stab->nextFrame();
    } else {
        Ptr<OnePassVideoStabilizer> stab = stabilizer_.dynamicCast<OnePassVideoStabilizer>();
        return stab->nextFrame();
    }
}

Mat
VideoStabilizer::nextStabilizedMotion(DvsData* frame, int& stablizedPos)
{
    Mat HMatrix;

    if (isTwoPass_) {
        Ptr<TwoPassVideoStabilizer> stab = stabilizer_.dynamicCast<TwoPassVideoStabilizer>();
        HMatrix = stab->nextStabilizedMotion(frame, stablizedPos);
    } else {
        Ptr<OnePassVideoStabilizer> stab = stabilizer_.dynamicCast<OnePassVideoStabilizer>();
        HMatrix = stab->nextStabilizedMotion(frame, stablizedPos);
    }

    return HMatrix;
}

Size
VideoStabilizer::trimedVideoSize(Size frameSize)
{
    Size outputFrameSize;
    outputFrameSize.width = ((int)((float)frameSize.width * (1 - 2 * trimRatio_)) >> 3) << 3;
    outputFrameSize.height = ((int)((float)frameSize.height * (1 - 2 * trimRatio_)) >> 3) << 3;

    return (outputFrameSize);
}

Mat
VideoStabilizer::cropVideoFrame(Mat& frame)
{
    Rect cropROI;
    Size inputFrameSize = frame.size();
    Size outputFrameSize = trimedVideoSize(inputFrameSize);

    cropROI.x = (inputFrameSize.width - outputFrameSize.width) >> 1;
    cropROI.y = (inputFrameSize.height - outputFrameSize.height) >> 1;
    cropROI.width = outputFrameSize.width;
    cropROI.height = outputFrameSize.height;

    Mat croppedFrame = frame(cropROI).clone();

    return croppedFrame;
}