Loading...
Searching...
No Matches
Beauty-Demo
Prev Tutorial: Optflow-Demo
Next Tutorial: V4D
Original author | Amir Hassan (kallaballa) amir@.nosp@m.viel.nosp@m.-zu.o.nosp@m.rg |
Compatibility | OpenCV >= 4.7 |
Face beautification using face landmark detection (OpenCV), nanovg (OpenGL) for drawing masks and multi-band blending to put it all together.
Downloading...
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright Amir Hassan (kallaballa) <amir@viel-zu.org>
#include <opencv2/v4d/v4d.hpp>
#include <opencv2/dnn.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/face.hpp>
#include <opencv2/stitching/detail/blenders.hpp>
#include <opencv2/tracking.hpp>
#include <vector>
#include <string>
using std::cerr;
using std::endl;
using std::vector;
using std::string;
/* Demo parameters */
#ifndef __EMSCRIPTEN__
constexpr long unsigned int WIDTH = 1280;
constexpr long unsigned int HEIGHT = 720;
#else
constexpr long unsigned int WIDTH = 960;
constexpr long unsigned int HEIGHT = 960;
#endif
constexpr unsigned int DOWNSIZE_WIDTH = 960;
constexpr unsigned int DOWNSIZE_HEIGHT = 540;
constexpr bool OFFSCREEN = false;
#ifndef __EMSCRIPTEN__
constexpr const char *OUTPUT_FILENAME = "beauty-demo.mkv";
#endif
const unsigned long DIAG = hypot(double(WIDTH), double(HEIGHT));
constexpr int BLUR_DIV = 500;
struct FaceFeatures {
cv::Rect faceRect_;
vector<cv::Point2f> chin_;
vector<cv::Point2f> top_nose_;
vector<cv::Point2f> bottom_nose_;
vector<cv::Point2f> left_eyebrow_;
vector<cv::Point2f> right_eyebrow_;
vector<cv::Point2f> left_eye_;
vector<cv::Point2f> right_eye_;
vector<cv::Point2f> outer_lips_;
vector<cv::Point2f> inside_lips_;
FaceFeatures() {};
//calculate the face rectangle
faceRect_ = cv::Rect(faceRect.x / local_scale, faceRect.y / local_scale, faceRect.width / local_scale, faceRect.height / local_scale);
size_t i = 0;
// Around Chin. Ear to Ear
for (i = 0; i <= 16; ++i)
chin_.push_back(shape[i] / local_scale);
// left eyebrow
for (; i <= 21; ++i)
left_eyebrow_.push_back(shape[i] / local_scale);
// Right eyebrow
for (; i <= 26; ++i)
right_eyebrow_.push_back(shape[i] / local_scale);
// Line on top of nose
for (; i <= 30; ++i)
top_nose_.push_back(shape[i] / local_scale);
// Bottom part of the nose
for (; i <= 35; ++i)
bottom_nose_.push_back(shape[i] / local_scale);
// Left eye
for (; i <= 41; ++i)
left_eye_.push_back(shape[i] / local_scale);
// Right eye
for (; i <= 47; ++i)
right_eye_.push_back(shape[i] / local_scale);
// Lips outer part
for (; i <= 59; ++i)
outer_lips_.push_back(shape[i] / local_scale);
// Lips inside part
for (; i <= 67; ++i)
inside_lips_.push_back(shape[i] / local_scale);
}
//Concatenates all feature points
vector<cv::Point2f> points() const {
vector<cv::Point2f> allPoints;
allPoints.insert(allPoints.begin(), chin_.begin(), chin_.end());
allPoints.insert(allPoints.begin(), top_nose_.begin(), top_nose_.end());
allPoints.insert(allPoints.begin(), bottom_nose_.begin(), bottom_nose_.end());
allPoints.insert(allPoints.begin(), left_eyebrow_.begin(), left_eyebrow_.end());
allPoints.insert(allPoints.begin(), right_eyebrow_.begin(), right_eyebrow_.end());
allPoints.insert(allPoints.begin(), left_eye_.begin(), left_eye_.end());
allPoints.insert(allPoints.begin(), right_eye_.begin(), right_eye_.end());
allPoints.insert(allPoints.begin(), outer_lips_.begin(), outer_lips_.end());
allPoints.insert(allPoints.begin(), inside_lips_.begin(), inside_lips_.end());
return allPoints;
}
//Returns all feature points in fixed order
vector<vector<cv::Point2f>> features() const {
return {chin_,
top_nose_,
bottom_nose_,
left_eyebrow_,
right_eyebrow_,
left_eye_,
right_eye_,
outer_lips_,
inside_lips_};
}
size_t empty() const {
return points().empty();
}
};
using namespace cv::v4d;
struct Params {
int blurSkinKernelSize_ = std::max(int(DIAG / BLUR_DIV % 2 == 0 ? DIAG / BLUR_DIV + 1 : DIAG / BLUR_DIV), 1);
//Saturation boost factor for eyes and lips
float eyesAndLipsSaturation_ = 2.0f;
//Saturation boost factor for skin
float skinSaturation_ = 1.7f;
//Contrast factor skin
float skinContrast_ = 0.7f;
#ifndef __EMSCRIPTEN__
//Show input and output side by side
bool sideBySide_ = true;
//Scale the video to the window size
bool stretch_ = true;
#else
bool sideBySide_ = false;
bool stretch_ = false;
#endif
} params_;
struct Cache {
vector<cv::UMat> channels_;
cv::UMat hls_;
} cache_;
cv::Ptr<cv::face::Facemark> facemark_ = cv::face::createFacemarkLBF();
//Blender (used to put the different face parts back together)
//Face detector
#ifndef __EMSCRIPTEN__
cv::Ptr<cv::FaceDetectorYN> detector_ = cv::FaceDetectorYN::create("modules/v4d/assets/models/face_detection_yunet_2023mar.onnx", "", cv::Size(DOWNSIZE_WIDTH, DOWNSIZE_HEIGHT), 0.9, 0.3, 5000, cv::dnn::DNN_BACKEND_OPENCV, cv::dnn::DNN_TARGET_OPENCL);
#else
cv::Ptr<cv::FaceDetectorYN> detector_ = cv::FaceDetectorYN::create("assets/models/face_detection_yunet_2023mar.onnx", "", cv::Size(DOWNSIZE_WIDTH, DOWNSIZE_HEIGHT), 0.9, 0.3, 5000, cv::dnn::DNN_BACKEND_OPENCV, cv::dnn::DNN_TARGET_CPU);
#endif
//BGR
cv::UMat input_, down_, contrast_, faceOval_, eyesAndLips_, skin_;
//GREY
cv::UMat faceSkinMaskGrey_, eyesAndLipsMaskGrey_, backgroundMaskGrey_;
//BGR-Float
//list all of shapes (face features) found
vector<vector<cv::Point2f>> shapes_;
std::vector<cv::Rect> faceRects_;
bool faceFound_ = false;
FaceFeatures features_;
public:
//based on the detected FaceFeatures it guesses a decent face oval and draws a mask for it.
static void draw_face_oval_mask(const FaceFeatures &ff) {
using namespace cv::v4d::nvg;
clear();
vector<vector<cv::Point2f>> features = ff.features();
cv::RotatedRect rotRect = cv::fitEllipse(features[0]);
beginPath();
fillColor(cv::Scalar(255, 255, 255, 255));
fill();
}
//Draws a mask consisting of eyes and lips areas (deduced from FaceFeatures)
static void draw_face_eyes_and_lips_mask(const FaceFeatures &ff) {
using namespace cv::v4d::nvg;
clear();
vector<vector<cv::Point2f>> features = ff.features();
for (size_t j = 5; j < 8; ++j) {
beginPath();
fillColor(cv::Scalar(255, 255, 255, 255));
moveTo(features[j][0].x, features[j][0].y);
for (size_t k = 1; k < features[j].size(); ++k) {
lineTo(features[j][k].x, features[j][k].y);
}
closePath();
fill();
}
beginPath();
fillColor(cv::Scalar(0, 0, 0, 255));
moveTo(features[8][0].x, features[8][0].y);
for (size_t k = 1; k < features[8].size(); ++k) {
lineTo(features[8][k].x, features[8][k].y);
}
closePath();
fill();
}
//adjusts the saturation of a UMat
static void adjust_saturation(const cv::UMat &srcBGR, cv::UMat &dstBGR, float factor, Cache& cache) {
cvtColor(srcBGR, cache.hls_, cv::COLOR_BGR2HLS);
split(cache.hls_, cache.channels_);
cv::multiply(cache.channels_[2], factor, cache.channels_[2]);
merge(cache.channels_, cache.hls_);
cvtColor(cache.hls_, dstBGR, cv::COLOR_HLS2BGR);
}
window->imgui([](cv::Ptr<V4D> win, ImGuiContext* ctx, Params& params){
using namespace ImGui;
SetCurrentContext(ctx);
Begin("Effect");
Text("Display");
Checkbox("Side by side", ¶ms.sideBySide_);
win->setStretching(true);
} else
win->setStretching(false);
#ifndef __EMSCRIPTEN__
if(Button("Fullscreen")) {
win->setFullscreen(!win->isFullscreen());
};
#endif
if(Button("Offscreen")) {
win->setVisible(!win->isVisible());
};
Text("Face Skin");
SliderInt("Blur", ¶ms.blurSkinKernelSize_, 0, 128);
SliderFloat("Saturation", ¶ms.skinSaturation_, 0.0f, 100.0f);
SliderFloat("Contrast", ¶ms.skinContrast_, 0.0f, 1.0f);
Text("Eyes and Lips");
SliderFloat("Saturation ", ¶ms.eyesAndLipsSaturation_, 0.0f, 100.0f);
End();
}, params_);
}
window->setStretching(params_.stretch_);
window->parallel([](cv::Ptr<cv::face::Facemark>& facemark){
#ifndef __EMSCRIPTEN__
facemark->loadModel("modules/v4d/assets/models/lbfmodel.yaml");
#else
facemark->loadModel("assets/models/lbfmodel.yaml");
#endif
cerr << "Loading finished" << endl;
}, facemark_);
}
auto always = [](){ return true; };
auto isTrue = [](bool& ff){ return ff; };
auto isFalse = [](bool& ff){ return !ff; };
try {
window->branch(always);
{
window->capture();
//Save the video frame as BGR
cvtColor(framebuffer, in, cv::COLOR_BGRA2BGR);
//Downscale the video frame for face detection
cv::resize(in, d, cv::Size(DOWNSIZE_WIDTH, DOWNSIZE_HEIGHT));
}, input_, down_);
window->parallel([](cv::Ptr<cv::FaceDetectorYN>& de, cv::Ptr<cv::face::Facemark>& fm, vector<vector<cv::Point2f>>& sh, const cv::UMat& d, std::vector<cv::Rect>& fr, bool& ff, FaceFeatures& ft) {
sh.clear();
cv::Mat faces;
//Detect faces in the down-scaled image
de->detect(d, faces);
//Only add the first face
cv::Rect faceRect;
if(!faces.empty())
faceRect = cv::Rect(int(faces.at<float>(0, 0)), int(faces.at<float>(0, 1)), int(faces.at<float>(0, 2)), int(faces.at<float>(0, 3)));
fr = {faceRect};
//find landmarks if faces have been detected
ff = !faceRect.empty() && fm->fit(d, fr, sh);
if(ff)
ft = FaceFeatures(fr[0], sh[0], float(d.size().width) / WIDTH);
}, detector_, facemark_, shapes_, down_, faceRects_, faceFound_, features_);
}
window->endbranch(always);
window->branch(isTrue, faceFound_);
{
window->nvg([](const FaceFeatures& f) {
//Draw the face oval of the first face
draw_face_oval_mask(f);
}, features_);
//Convert/Copy the mask
cvtColor(frameBuffer, fo, cv::COLOR_BGRA2GRAY);
}, faceOval_);
window->nvg([](const FaceFeatures& f) {
//Draw eyes eyes and lips areas of the first face
draw_face_eyes_and_lips_mask(f);
}, features_);
//Convert/Copy the mask
cvtColor(frameBuffer, ealmg, cv::COLOR_BGRA2GRAY);
}, eyesAndLipsMaskGrey_);
//Create the skin mask
cv::subtract(fo, ealmg, fsmg);
//Create the background mask
cv::bitwise_not(ealmg, bmg);
}, faceOval_, eyesAndLipsMaskGrey_, faceSkinMaskGrey_, backgroundMaskGrey_);
window->parallel([](const cv::UMat& in, cv::UMat& eal, cv::UMat& c, cv::UMat& s, Params& params, Cache& cache) {
//boost saturation of eyes and lips
adjust_saturation(in, eal, params.eyesAndLipsSaturation_, cache);
//reduce skin contrast
//fix skin brightness
//blur the skin_
cv::boxFilter(c, c, -1, cv::Size(params.blurSkinKernelSize_, params.blurSkinKernelSize_), cv::Point(-1, -1), true, cv::BORDER_REPLICATE);
//boost skin saturation
adjust_saturation(c, s, params.skinSaturation_, cache);
}, input_, eyesAndLips_, contrast_, skin_, params_, cache_);
window->parallel([](cv::Ptr<cv::detail::MultiBandBlender>& bl,
cv::UMat& fout) {
cv:: UMat foFloat;
//FIXME do it once?
//piece it all together
//FIXME prepare only once?
bl->prepare(cv::Rect(0, 0, WIDTH, HEIGHT));
bl->feed(s, fsmg, cv::Point(0, 0));
bl->feed(in, bmg, cv::Point(0, 0));
bl->feed(eal, ealmg, cv::Point(0, 0));
bl->blend(foFloat, cv::UMat());
CV_Assert(!foFloat.empty());
foFloat.convertTo(fout, CV_8U, 1.0);
}, blender_, skin_, faceSkinMaskGrey_, input_, backgroundMaskGrey_, eyesAndLips_, eyesAndLipsMaskGrey_, frameOut_);
window->parallel([](cv::UMat& fout, const cv::UMat& in, cv::UMat& lh, cv::UMat& rh, const Params& params) {
if (params.sideBySide_) {
//create side-by-side view with a result
cv::resize(in, lh, cv::Size(0, 0), 0.5, 0.5);
cv::resize(fout, rh, cv::Size(0, 0), 0.5, 0.5);
fout = cv::Scalar::all(0);
}
}, frameOut_, input_, lhalf_, rhalf_, params_);
}
window->endbranch(isTrue, faceFound_);
window->branch(isFalse, faceFound_);
{
if (params.sideBySide_) {
//create side-by-side view without a result (using the input image for both sides)
fout = cv::Scalar::all(0);
cv::resize(in, lh, cv::Size(0, 0), 0.5, 0.5);
} else {
in.copyTo(fout);
}
}, frameOut_, input_, lhalf_, params_);
}
window->endbranch(isFalse, faceFound_);
window->branch(always);
{
//write the result to the framebuffer
cvtColor(f, frameBuffer, cv::COLOR_BGR2BGRA);
}, frameOut_);
}
window->endbranch(always);
} catch (std::exception &ex) {
cerr << ex.what() << endl;
}
}
};
#ifndef __EMSCRIPTEN__
int main(int argc, char **argv) {
if (argc != 2) {
cerr << "Usage: beauty-demo <input-video-file>" << endl;
exit(1);
}
#else
int main() {
#endif
using namespace cv::v4d;
cv::Ptr<V4D> window = V4D::make(WIDTH, HEIGHT, "Beautification Demo", ALL, OFFSCREEN);
#ifndef __EMSCRIPTEN__
auto src = makeCaptureSource(window, argv[1]);
window->setSource(src);
window->setSink(sink);
#else
auto src = makeCaptureSource(window);
window->setSource(src);
#endif
window->run<BeautyDemoPlan>(0);
return 0;
}
static Ptr< FaceDetectorYN > create(const String &model, const String &config, const Size &input_size, float score_threshold=0.9f, float nms_threshold=0.3f, int top_k=5000, int backend_id=0, int target_id=0)
Creates an instance of this class with given parameters.
The class represents rotated (i.e. not up-right) rectangles on a plane.
Definition: types.hpp:531
float angle
returns the rotation angle. When the angle is 0, 90, 180, 270 etc., the rectangle becomes an up-right...
Definition: types.hpp:567
static Scalar_< double > all(double v0)
returns a scalar with all elements set to v0
Definition: mat.hpp:2432
MatSize size
dimensional size of the matrix; accessible in various formats
Definition: mat.hpp:2643
Blender which uses multi-band blending algorithm (see ).
Definition: blenders.hpp:128
Definition: v4d.hpp:68
void bitwise_not(InputArray src, OutputArray dst, InputArray mask=noArray())
Inverts every bit of an array.
void split(const Mat &src, Mat *mvbegin)
Divides a multi-channel array into several single-channel arrays.
void add(InputArray src1, InputArray src2, OutputArray dst, InputArray mask=noArray(), int dtype=-1)
Calculates the per-element sum of two arrays or an array and a scalar.
void rotate(InputArray src, OutputArray dst, int rotateCode)
Rotates a 2D array in multiples of 90 degrees. The function cv::rotate rotates the array in one of th...
void merge(const Mat *mv, size_t count, OutputArray dst)
Creates one multi-channel array out of several single-channel ones.
void multiply(InputArray src1, InputArray src2, OutputArray dst, double scale=1, int dtype=-1)
Calculates the per-element scaled product of two arrays.
void subtract(InputArray src1, InputArray src2, OutputArray dst, InputArray mask=noArray(), int dtype=-1)
Calculates the per-element difference between two arrays or array and a scalar.
#define CV_Assert(expr)
Checks a condition at runtime and throws exception if it fails.
Definition: base.hpp:342
void cvtColor(InputArray src, OutputArray dst, int code, int dstCn=0)
Converts an image from one color space to another.
@ COLOR_HLS2BGR
backward conversions HLS to RGB/BGR with H range 0..180 if 8 bit image
Definition: imgproc.hpp:616
@ COLOR_BGR2HLS
convert RGB/BGR to HLS (hue lightness saturation) with H range 0..180 if 8 bit image,...
Definition: imgproc.hpp:606
@ COLOR_BGRA2BGR
remove alpha channel from RGB or BGR image
Definition: imgproc.hpp:540
void ellipse(InputOutputArray img, Point center, Size axes, double angle, double startAngle, double endAngle, const Scalar &color, int thickness=1, int lineType=LINE_8, int shift=0)
Draws a simple or thick elliptic arc or fills an ellipse sector.
void boxFilter(InputArray src, OutputArray dst, int ddepth, Size ksize, Point anchor=Point(-1,-1), bool normalize=true, int borderType=BORDER_DEFAULT)
Blurs an image using the box filter.
void resize(InputArray src, OutputArray dst, Size dsize, double fx=0, double fy=0, int interpolation=INTER_LINEAR)
Resizes an image.
PyParams params(const std::string &tag, const std::string &model, const std::string &weights, const std::string &device)
Net::Result infer(cv::GOpaque< cv::Rect > roi, T in)
Calculates response for the specified network (template parameter) for the specified region in the so...
Definition: infer.hpp:474
Definition: nvg.hpp:20
void beginPath()
void lineTo(float x, float y)
void fillColor(const cv::Scalar &color)
void clear(const cv::Scalar &bgra=cv::Scalar(0, 0, 0, 255))
void closePath()
void moveTo(float x, float y)
void fill()
Definition: backend.hpp:15
cv::Ptr< Sink > makeWriterSink(cv::Ptr< V4D > window, const string &outputFilename, const float fps, const cv::Size &frameSize)
cv::Ptr< Source > makeCaptureSource(cv::Ptr< V4D > window, const string &inputFilename)