i want to do linear blending for my image stitching pipeline. My main reference for linear blending is this post: Blending does not remove seams in OpenCV It works on my stitching pipeline, but the result show ghosting in parts of the images. Here's my code
#include <iostream>
#include <string>
#include <algorithm>
#include <chrono>
#include "opencv2/opencv.hpp"
#include "opencv2/opencv_modules.hpp"
#include "opencv2/core/utility.hpp"
#include "opencv2/imgcodecs.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/xfeatures2d.hpp"
#include "opencv2/xfeatures2d/nonfree.hpp"
#include "opencv2/xfeatures2d/cuda.hpp"
#include "opencv2/cudafeatures2d.hpp"
#include "opencv2/cudaarithm.hpp"
cv::Mat border(cv::Mat mask)
{
cv::Mat gx;
cv::Mat gy;
cv::Sobel(mask, gx, CV_32F, 1, 0, 3);
cv::Sobel(mask, gy, CV_32F, 0, 1, 3);
cv::Mat border;
cv::magnitude(gx, gy, border);
return border > 100;
}
cv::Mat linearBlend2(cv::Mat image1, cv::Mat mask1, cv::Mat image2, cv::Mat mask2)
{
cv::TickMeter tm;
// === Init variable ===
cv::Mat distResult, distMask1, distMask2, diskMaskSum, borderMask;
cv::Mat imBlendedB, imBlendedG, imBlendedR, imgResult;
double min, max;
cv::Point minLoc, maxLoc;
cv::Mat im1Float, im2Float;
std::vector<cv::Mat> channels1, channels2, channelsBlended;
// edited: find regions where no mask is set
// compute the region where no mask is set at all, to use those color values unblended
tm.start();
cv::Mat bothMasks = mask1 | mask2;
cv::Mat noMask = 255 - bothMasks;
// create an image with equal alpha values:
cv::Mat rawAlpha = cv::Mat(noMask.rows, noMask.cols, CV_32FC1);
rawAlpha = 1.0f;
tm.stop();
std::cout << "Create mask = " << tm.getTimeMilli() << " ms\n";
// === 1. Process Image 1 ===
// invert the border, so that border values are 0 ... this is needed for the distance transform
borderMask = 255 - border(mask1);
// === a. Distance Transfrom ===
tm.start();
cv::distanceTransform(borderMask, distResult, cv::DIST_L2, 3);
tm.stop();
std::cout << "Distance mask 0 = " << tm.getTimeMilli() << " ms\n";
cv::imwrite("DistanceMask0.jpg", distResult);
// === b. scale distances to values between 0 and 1 ===
tm.start();
cv::minMaxLoc(distResult, &min, &max, &minLoc, &maxLoc, mask1 & (distResult > 0)); // edited: find min values > 0
distResult = distResult * 1.0 / max; // values between 0 and 1 since min val should alwaysbe 0
tm.stop();
std::cout << "Scale Distance mask 0 = " << tm.getTimeMilli() << " ms\n";
// === c. mask the distance values to reduce information to masked regions ===
tm.start();
rawAlpha.copyTo(distMask1, noMask); // edited: where no mask is set, blend with equal values
distResult.copyTo(distMask1, mask1);
rawAlpha.copyTo(distMask1, mask1 & (255 - mask2)); // edited
tm.stop();
std::cout << "Copy Distance mask 0 = " << tm.getTimeMilli() << " ms\n";
// === 2. Process Image 2 ===
borderMask = 255 - border(mask2);
// === a. Distance Transfrom ===
cv::distanceTransform(borderMask, distResult, cv::DIST_L2, 3);
cv::imwrite("DistanceMask1.jpg", distResult);
// === b. scale distances to values between 0 and 1 ===
cv::minMaxLoc(distResult, &min, &max, &minLoc, &maxLoc, mask2 & (distResult > 0)); // edited: find min values > 0
distResult = distResult * 1.0 / max; // values between 0 and 1
// === c. mask the distance values to reduce information to masked regions ===
rawAlpha.copyTo(distMask2, noMask); // edited: where no mask is set, blend with equal values
distResult.copyTo(distMask2, mask2);
rawAlpha.copyTo(distMask2, mask2 & (255 - mask1)); // edited
// === 3. Combine / blend both image ===
diskMaskSum = distMask1 + distMask2;
// you have to convert the images to float to multiply with the weight
tm.start();
image1.convertTo(im1Float, distMask1.type());
image2.convertTo(im2Float, distMask2.type());
cv::split(im1Float, channels1);
cv::split(im2Float, channels2);
tm.stop();
std::cout << "Split Image Channels = " << tm.getTimeMilli() << " ms\n";
cv::Mat im1Alpha;
std::vector<cv::Mat> alpha1;
cv::Mat im1AlphaB = distMask1.mul(channels1[0]);
cv::Mat im1AlphaG = distMask1.mul(channels1[1]);
cv::Mat im1AlphaR = distMask1.mul(channels1[2]);
alpha1.push_back(im1AlphaB);
alpha1.push_back(im1AlphaG);
alpha1.push_back(im1AlphaR);
cv::merge(alpha1, im1Alpha);
cv::imshow("alpha1", im1Alpha / 255.0);
cv::imwrite("AppliedMask0.jpg", im1Alpha);
std::vector<cv::Mat> alpha2;
cv::Mat im2Alpha;
cv::Mat im2AlphaB = distMask2.mul(channels2[0]);
cv::Mat im2AlphaG = distMask2.mul(channels2[1]);
cv::Mat im2AlphaR = distMask2.mul(channels2[2]);
alpha2.push_back(im2AlphaB);
alpha2.push_back(im2AlphaG);
alpha2.push_back(im2AlphaR);
cv::merge(alpha2, im2Alpha);
cv::imshow("alpha2", im2Alpha / 255.0);
cv::imwrite("AppliedMask1.jpg", im2Alpha);
// now sum both weighphted images and divide by the sum of the weights (linear combination)
imBlendedB = (im1AlphaB + im2AlphaB) / diskMaskSum;
imBlendedG = (im1AlphaG + im2AlphaG) / diskMaskSum;
imBlendedR = (im1AlphaR + im2AlphaR) / diskMaskSum;
channelsBlended.push_back(imBlendedB);
channelsBlended.push_back(imBlendedG);
channelsBlended.push_back(imBlendedR);
// merge back to 3 channel image
cv::Mat merged;
cv::merge(channelsBlended, merged);
// convert to 8UC3
cv::Mat merged8U;
merged.convertTo(merged8U, CV_8UC3);
return merged8U;
}
cv::Mat StitchImages()
{
/*
* Image stitching pipeline:
* */
// create vector and allocate it with 2 images
std::vector<cv::Mat> imagesWarpedArray(2);
std::vector<cv::Mat> maskWarpedArray(2);
imagesWarpedArray[0] = cv::imread("../imagesWarpedArray0.jpg");
imagesWarpedArray[1] = cv::imread("../imagesWarpedArray1.jpg");
maskWarpedArray[0] = cv::imread("../maskWarpedArray0.jpg");
maskWarpedArray[1] = cv::imread("../maskWarpedArray0.jpg");
maskWarpedArray[0] = warpedMaskBase;
maskWarpedArray[1] = warpedMaskSec;
// blend image
cv::Mat blendResult = linearBlend2(imagesWarpedArray[0], maskWarpedArray[0], imagesWarpedArray[1], maskWarpedArray[1]);
cv::imshow("Result", blendResult);
return blendResult;
}
int main(int argc, char **argv)
{
cv::Mat result = StitchImages();
cv::waitKey(0);
return 0;
}
Image that i used
ImageWarped[1]
ImageWarped[0]
maskWarped[0]
maskWarped[1]
Result:
Panorama_Result
Well, i know that OpenCV has auto stitching library but it's not possible for me since the algorithm run too slow for video input (direct camera stream).
Thanks in advance