No Matches

Prev Tutorial: Pedestrian-Demo
Next Tutorial: Beauty-Demo

Original author Amir Hassan (kallaballa) amir@.nosp@m.viel.nosp@m.-zu.o.nosp@m.rg
Compatibility OpenCV >= 4.7

Optical flow visualization on top of a video. Uses background subtraction (OpenCV) to isolate areas with motion, detects features to track (OpenCV), calculates the optical flow (OpenCV), uses nanovg for rendering (OpenGL) and post-processes the video (OpenCV).

Sparse Optical Flow Demo
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright Amir Hassan (kallaballa) <amir@viel-zu.org>
#include <cmath>
#include <vector>
#include <set>
#include <string>
#include <random>
using std::cerr;
using std::endl;
using std::vector;
using std::string;
/* Demo parameters */
#ifndef __EMSCRIPTEN__
constexpr long unsigned int WIDTH = 1280;
constexpr long unsigned int HEIGHT = 720;
constexpr long unsigned int WIDTH = 960;
constexpr long unsigned int HEIGHT = 960;
const unsigned long DIAG = hypot(double(WIDTH), double(HEIGHT));
#ifndef __EMSCRIPTEN__
constexpr const char* OUTPUT_FILENAME = "optflow-demo.mkv";
constexpr bool OFFSCREEN = false;
#ifndef __EMSCRIPTEN__
//the second window
static cv::Ptr<cv::v4d::V4D> miniWindow;
/* Visualization parameters */
//How the background will be visualized
enum BackgroundModes {
//Post-processing modes for the foreground
enum PostProcModes {
// Generate the foreground at this scale.
float fg_scale = 0.5f;
// On every frame the foreground loses on brightness. Specifies the loss in percent.
#ifndef __EMSCRIPTEN__
float fg_loss = 2.5;
float fg_loss = 10.0;
//Convert the background to greyscale
BackgroundModes background_mode = GREY;
// Peak thresholds for the scene change detection. Lowering them makes the detection more sensitive but
// the default should be fine.
float scene_change_thresh = 0.29f;
float scene_change_thresh_diff = 0.1f;
// The theoretical maximum number of points to track which is scaled by the density of detected points
// and therefor is usually much smaller.
#ifndef __EMSCRIPTEN__
int max_points = 250000;
int max_points = 100000;
// How many of the tracked points to lose intentionally, in percent.
#ifndef __EMSCRIPTEN__
float point_loss = 25;
float point_loss = 10;
// The theoretical maximum size of the drawing stroke which is scaled by the area of the convex hull
// of tracked points and therefor is usually much smaller.
int max_stroke = 10;
// Red, green, blue and alpha. All from 0.0f to 1.0f
float effect_color[4] = {1.0f, 0.75f, 0.4f, 1.0f};
//display on-screen FPS
bool show_fps = true;
//Stretch frame buffer to window size
bool stretch = false;
//Use OpenCL or not
bool use_acceleration = true;
//The post processing mode
#ifndef __EMSCRIPTEN__
PostProcModes post_proc_mode = GLOW;
PostProcModes post_proc_mode = NONE;
// Intensity of glow or bloom defined by kernel size. The default scales with the image diagonal.
int glow_kernel_size = std::max(int(DIAG / 100 % 2 == 0 ? DIAG / 100 + 1 : DIAG / 100), 1);
//The lightness selection threshold
int bloom_thresh = 210;
//The intensity of the bloom filter
float bloom_gain = 3;
//Uses background subtraction to generate a "motion mask"
static void prepare_motion_mask(const cv::UMat& srcGrey, cv::UMat& motionMaskGrey) {
static int morph_size = 1;
static cv::Mat element = cv::getStructuringElement(cv::MORPH_RECT, cv::Size(2 * morph_size + 1, 2 * morph_size + 1), cv::Point(morph_size, morph_size));
bg_subtrator->apply(srcGrey, motionMaskGrey);
//Surpress speckles
cv::morphologyEx(motionMaskGrey, motionMaskGrey, cv::MORPH_OPEN, element, cv::Point(element.cols >> 1, element.rows >> 1), 2, cv::BORDER_CONSTANT, cv::morphologyDefaultBorderValue());
//Detect points to track
static void detect_points(const cv::UMat& srcMotionMaskGrey, vector<cv::Point2f>& points) {
static vector<cv::KeyPoint> tmpKeyPoints;
detector->detect(srcMotionMaskGrey, tmpKeyPoints);
for (const auto &kp : tmpKeyPoints) {
//Detect extrem changes in scene content and report it
static bool detect_scene_change(const cv::UMat& srcMotionMaskGrey, const float thresh, const float theshDiff) {
static float last_movement = 0;
float movement = cv::countNonZero(srcMotionMaskGrey) / float(srcMotionMaskGrey.cols * srcMotionMaskGrey.rows);
float relation = movement > 0 && last_movement > 0 ? std::max(movement, last_movement) / std::min(movement, last_movement) : 0;
float relM = relation * log10(1.0f + (movement * 9.0));
float relLM = relation * log10(1.0f + (last_movement * 9.0));
bool result = !((movement > 0 && last_movement > 0 && relation > 0)
&& (relM < thresh && relLM < thresh && fabs(relM - relLM) < theshDiff));
last_movement = (last_movement + movement) / 2.0f;
return result;
//Visualize the sparse optical flow
static void visualize_sparse_optical_flow(const cv::UMat &prevGrey, const cv::UMat &nextGrey, const vector<cv::Point2f> &detectedPoints, const float scaleFactor, const int maxStrokeSize, const cv::Scalar color, const int maxPoints, const float pointLossPercent) {
static vector<cv::Point2f> hull, prevPoints, nextPoints, newPoints;
static vector<cv::Point2f> upPrevPoints, upNextPoints;
static std::vector<uchar> status;
static std::vector<float> err;
static std::random_device rd;
static std::mt19937 g(rd());
//less then 5 points is a degenerate case (e.g. the corners of a video frame)
if (detectedPoints.size() > 4) {
cv::convexHull(detectedPoints, hull);
float area = cv::contourArea(hull);
//make sure the area of the point cloud is positive
if (area > 0) {
float density = (detectedPoints.size() / area);
//stroke size is biased by the area of the point cloud
float strokeSize = maxStrokeSize * pow(area / (nextGrey.cols * nextGrey.rows), 0.33f);
//max points is biased by the densitiy of the point cloud
size_t currentMaxPoints = ceil(density * maxPoints);
//lose a number of random points specified by pointLossPercent
std::shuffle(prevPoints.begin(), prevPoints.end(), g);
prevPoints.resize(ceil(prevPoints.size() * (1.0f - (pointLossPercent / 100.0f))));
//calculate how many newly detected points to add
size_t copyn = std::min(detectedPoints.size(), (size_t(std::ceil(currentMaxPoints)) - prevPoints.size()));
if (prevPoints.size() < currentMaxPoints) {
std::copy(detectedPoints.begin(), detectedPoints.begin() + copyn, std::back_inserter(prevPoints));
//calculate the sparse optical flow
cv::calcOpticalFlowPyrLK(prevGrey, nextGrey, prevPoints, nextPoints, status, err);
if (prevPoints.size() > 1 && nextPoints.size() > 1) {
//scale the points to original size
for (cv::Point2f pt : prevPoints) {
upPrevPoints.push_back(pt /= scaleFactor);
for (cv::Point2f pt : nextPoints) {
upNextPoints.push_back(pt /= scaleFactor);
using namespace cv::v4d::nvg;
//start drawing
for (size_t i = 0; i < prevPoints.size(); i++) {
if (status[i] == 1 //point was found in prev and new set
&& err[i] < (1.0 / density) //with a higher density be more sensitive to the feature error
&& upNextPoints[i].y >= 0 && upNextPoints[i].x >= 0 //check bounds
&& upNextPoints[i].y < nextGrey.rows / scaleFactor && upNextPoints[i].x < nextGrey.cols / scaleFactor //check bounds
) {
float len = hypot(fabs(upPrevPoints[i].x - upNextPoints[i].x), fabs(upPrevPoints[i].y - upNextPoints[i].y));
//upper and lower bound of the flow vector lengthss
if (len > 0 && len < sqrt(area)) {
//collect new points
//the actual drawing operations
moveTo(upNextPoints[i].x, upNextPoints[i].y);
lineTo(upPrevPoints[i].x, upPrevPoints[i].y);
//end drawing
prevPoints = newPoints;
//Bloom post-processing effect
static void bloom(const cv::UMat& src, cv::UMat &dst, int ksize = 3, int threshValue = 235, float gain = 4) {
static cv::UMat bgr;
static cv::UMat hls;
static cv::UMat ls16;
static cv::UMat ls;
static cv::UMat blur;
static std::vector<cv::UMat> hlsChannels;
//remove alpha channel
//convert to hls
//split channels
cv::split(hls, hlsChannels);
//invert lightness
cv::bitwise_not(hlsChannels[2], hlsChannels[2]);
//multiply lightness and saturation
cv::multiply(hlsChannels[1], hlsChannels[2], ls16, 1, CV_16U);
cv::divide(ls16, cv::Scalar(255.0), ls, 1, CV_8U);
//binary threhold according to threshValue
cv::threshold(ls, blur, threshValue, 255, cv::THRESH_BINARY);
cv::boxFilter(blur, blur, -1, cv::Size(ksize, ksize), cv::Point(-1,-1), true, cv::BORDER_REPLICATE);
//convert to BGRA
//add src and the blurred L-S-product according to gain
addWeighted(src, 1.0, blur, gain, 0, dst);
//Glow post-processing effect
static void glow_effect(const cv::UMat &src, cv::UMat &dst, const int ksize) {
static cv::UMat resize;
static cv::UMat blur;
static cv::UMat dst16;
cv::bitwise_not(src, dst);
//Resize for some extra performance
cv::resize(dst, resize, cv::Size(), 0.5, 0.5);
//Cheap blur
cv::boxFilter(resize, resize, -1, cv::Size(ksize, ksize), cv::Point(-1,-1), true, cv::BORDER_REPLICATE);
//Back to original size
cv::resize(resize, blur, src.size());
//Multiply the src image with a blurred version of itself
cv::multiply(dst, blur, dst16, 1, CV_16U);
//Normalize and convert back to CV_8U
cv::divide(dst16, cv::Scalar::all(255.0), dst, 1, CV_8U);
cv::bitwise_not(dst, dst);
//Compose the different layers into the final image
static void composite_layers(cv::UMat& background, const cv::UMat& foreground, const cv::UMat& frameBuffer, cv::UMat& dst, int kernelSize, float fgLossPercent, BackgroundModes bgMode, PostProcModes ppMode) {
static cv::UMat tmp;
static cv::UMat post;
static cv::UMat backgroundGrey;
static vector<cv::UMat> channels;
//Lose a bit of foreground brightness based on fgLossPercent
cv::subtract(foreground, cv::Scalar::all(255.0f * (fgLossPercent / 100.0f)), foreground);
//Add foreground an the current framebuffer into foregound
cv::add(foreground, frameBuffer, foreground);
//Dependin on bgMode prepare the background in different ways
switch (bgMode) {
case GREY:
cv::cvtColor(background, backgroundGrey, cv::COLOR_BGRA2GRAY);
cv::cvtColor(backgroundGrey, background, cv::COLOR_GRAY2BGRA);
case VALUE:
cv::cvtColor(background, tmp, cv::COLOR_BGRA2BGR);
split(tmp, channels);
cv::cvtColor(channels[2], background, cv::COLOR_GRAY2BGRA);
case COLOR:
case BLACK:
background = cv::Scalar::all(0);
//Depending on ppMode perform post-processing
switch (ppMode) {
case GLOW:
glow_effect(foreground, post, kernelSize);
case BLOOM:
bloom(foreground, post, kernelSize, bloom_thresh, bloom_gain);
case NONE:
//Add background and post-processed foreground into dst
cv::add(background, post, dst);
using namespace cv::v4d;
//Build the GUI
static void setup_gui(cv::Ptr<V4D> main, cv::Ptr<V4D> mini) {
main->imgui([main](ImGuiContext* ctx){
using namespace ImGui;
SliderFloat("Scale", &fg_scale, 0.1f, 4.0f);
SliderFloat("Loss", &fg_loss, 0.1f, 99.9f);
static const char* bgm_items[4] = {"Grey", "Color", "Value", "Black"};
static int* bgm = (int*)&background_mode;
ListBox("Mode", bgm, bgm_items, 4, 4);
SliderInt("Max. Points", &max_points, 10, 1000000);
SliderFloat("Point Loss", &point_loss, 0.0f, 100.0f);
Text("Optical flow");
SliderInt("Max. Stroke Size", &max_stroke, 1, 100);
ColorPicker4("Color", effect_color);
Begin("Post Processing");
static const char* ppm_items[3] = {"Glow", "Bloom", "None"};
static int* ppm = (int*)&post_proc_mode;
ListBox("Effect",ppm, ppm_items, 3, 3);
SliderInt("Kernel Size",&glow_kernel_size, 1, 63);
SliderFloat("Gain", &bloom_gain, 0.1f, 20.0f);
Text("Scene Change Detection");
SliderFloat("Threshold", &scene_change_thresh, 0.1f, 1.0f);
SliderFloat("Threshold Diff", &scene_change_thresh_diff, 0.1f, 1.0f);
#ifndef __EMSCRIPTEN__
mini->imgui([main, mini](ImGuiContext* ctx){
using namespace ImGui;
if(Checkbox("Show FPS", &show_fps)) {
#ifndef __EMSCRIPTEN__
if(Checkbox("Stretch", &stretch)) {
#ifndef __EMSCRIPTEN__
if(Button("Fullscreen")) {
if(Button("Offscreen")) {
static bool iteration(cv::Ptr<V4D> window) {
static cv::UMat background, down;
static cv::UMat foreground(window->framebufferSize(), CV_8UC4, cv::Scalar::all(0));
static cv::UMat miniFrame;
static cv::UMat downPrevGrey, downNextGrey, downMotionMaskGrey;
static vector<cv::Point2f> detectedPoints;
return false;
window->fb([&](cv::UMat& frameBuffer) {
//resize to foreground scale
cv::resize(frameBuffer, down, cv::Size(window->framebufferSize().width * fg_scale, window->framebufferSize().height * fg_scale));
//save video background
cv::cvtColor(down, downNextGrey, cv::COLOR_RGBA2GRAY);
//Subtract the background to create a motion mask
prepare_motion_mask(downNextGrey, downMotionMaskGrey);
//Detect trackable points in the motion mask
detect_points(downMotionMaskGrey, detectedPoints);
window->nvg([&]() {
if (!downPrevGrey.empty()) {
//We don't want the algorithm to get out of hand when there is a scene change, so we suppress it when we detect one.
if (!detect_scene_change(downMotionMaskGrey, scene_change_thresh, scene_change_thresh_diff)) {
//Visualize the sparse optical flow using nanovg
cv::Scalar color = cv::Scalar(effect_color[2] * 255, effect_color[1] * 255, effect_color[0] * 255, effect_color[3] * 255);
visualize_sparse_optical_flow(downPrevGrey, downNextGrey, detectedPoints, fg_scale, max_stroke, color, max_points, point_loss);
downPrevGrey = downNextGrey.clone();
window->fb([&](cv::UMat& framebuffer){
//Put it all together (OpenCL)
composite_layers(background, foreground, framebuffer, framebuffer, glow_kernel_size, fg_loss, background_mode, post_proc_mode);
cvtColor(framebuffer, miniFrame, cv::COLOR_BGRA2RGB);
#ifndef __EMSCRIPTEN__
//If onscreen rendering is enabled it displays the framebuffer in the native window. Returns false if the window was closed.
#ifndef __EMSCRIPTEN__
if(window->isFocused()) {
return window->display() && miniWindow->display();
else {
return miniWindow->display() && window->display();
return window->display();
int main(int argc, char **argv) {
#ifndef __EMSCRIPTEN__
if (argc != 2) {
std::cerr << "Usage: optflow <input-video-file>" << endl;
try {
using namespace cv::v4d;
cv::Ptr<V4D> window = V4D::make(WIDTH, HEIGHT, "Sparse Optical Flow Demo", OFFSCREEN, false, 0);
#ifndef __EMSCRIPTEN__
miniWindow = V4D::make(270, 240, "Mini", OFFSCREEN, false, 0);
#ifndef __EMSCRIPTEN__
setup_gui(window, miniWindow);
setup_gui(window, window);
#ifndef __EMSCRIPTEN__
Source src = makeCaptureSource(argv[1]);
//Creates a writer sink (which might be hardware accelerated)
Sink sink = makeWriterSink(OUTPUT_FILENAME, src.fps(), cv::Size(WIDTH, HEIGHT));
Source src = makeCaptureSource(WIDTH, HEIGHT, window);
} catch (std::exception& ex) {
cerr << ex.what() << endl;
return 0;
static Ptr< FastFeatureDetector > create(int threshold=10, bool nonmaxSuppression=true, FastFeatureDetector::DetectorType type=FastFeatureDetector::TYPE_9_16)
n-dimensional dense array class
Definition: mat.hpp:811
int cols
Definition: mat.hpp:2137
int rows
the number of rows and columns or (-1, -1) when the matrix has more than 2 dimensions
Definition: mat.hpp:2137
static Scalar_< double > all(double v0)
returns a scalar with all elements set to v0
Template class for specifying the size of an image or rectangle.
Definition: types.hpp:335
Definition: mat.hpp:2432
int cols
number of columns in the matrix; -1 when the matrix has more than 2 dimensions
Definition: mat.hpp:2622
CV_NODISCARD_STD UMat clone() const
returns deep copy of the matrix, i.e. the data is copied
MatSize size
dimensional size of the matrix; accessible in various formats
Definition: mat.hpp:2643
int rows
number of rows in the matrix; -1 when the matrix has more than 2 dimensions
Definition: mat.hpp:2619
bool empty() const
returns true if matrix data is NULL
void copyTo(OutputArray m) const
copies the matrix content to "m".
Definition: sink.hpp:19
Definition: source.hpp:20
void bitwise_not(InputArray src, OutputArray dst, InputArray mask=noArray())
Inverts every bit of an array.
void split(const Mat &src, Mat *mvbegin)
Divides a multi-channel array into several single-channel arrays.
void add(InputArray src1, InputArray src2, OutputArray dst, InputArray mask=noArray(), int dtype=-1)
Calculates the per-element sum of two arrays or an array and a scalar.
void sqrt(InputArray src, OutputArray dst)
Calculates a square root of array elements.
void divide(InputArray src1, InputArray src2, OutputArray dst, double scale=1, int dtype=-1)
Performs per-element division of two arrays or a scalar by an array.
void multiply(InputArray src1, InputArray src2, OutputArray dst, double scale=1, int dtype=-1)
Calculates the per-element scaled product of two arrays.
void subtract(InputArray src1, InputArray src2, OutputArray dst, InputArray mask=noArray(), int dtype=-1)
Calculates the per-element difference between two arrays or array and a scalar.
int countNonZero(InputArray src)
Counts non-zero array elements.
void pow(InputArray src, double power, OutputArray dst)
Raises every array element to a power.
void addWeighted(InputArray src1, double alpha, InputArray src2, double beta, double gamma, OutputArray dst, int dtype=-1)
Calculates the weighted sum of two arrays.
Definition: base.hpp:270
iiiiii|abcdefgh|iiiiiii with some specified i
Definition: base.hpp:269
Scalar_< double > Scalar
Definition: types.hpp:702
std::shared_ptr< _Tp > Ptr
Definition: cvstd_wrapper.hpp:23
#define CV_8U
Definition: interface.h:73
#define CV_8UC4
Definition: interface.h:91
#define CV_16U
Definition: interface.h:75
__device__ __forceinline__ float1 hypot(const uchar1 &a, const uchar1 &b)
Definition: vec_math.hpp:803
__device__ __forceinline__ float1 log10(const uchar1 &a)
Definition: vec_math.hpp:276
void cvtColor(InputArray src, OutputArray dst, int code, int dstCn=0)
Converts an image from one color space to another.
convert RGB/BGR to HLS (hue lightness saturation) with H range 0..180 if 8 bit image,...
Definition: imgproc.hpp:606
remove alpha channel from RGB or BGR image
Definition: imgproc.hpp:540
Definition: imgproc.hpp:547
Definition: imgproc.hpp:562
convert RGB/BGR to HSV (hue saturation value) with H range 0..180 if 8 bit image, color conversions
Definition: imgproc.hpp:598
Definition: imgproc.hpp:559
Definition: imgproc.hpp:561
void morphologyEx(InputArray src, OutputArray dst, int op, InputArray kernel, Point anchor=Point(-1,-1), int iterations=1, int borderType=BORDER_CONSTANT, const Scalar &borderValue=morphologyDefaultBorderValue())
Performs advanced morphological transformations.
void blur(InputArray src, OutputArray dst, Size ksize, Point anchor=Point(-1,-1), int borderType=BORDER_DEFAULT)
Blurs an image using the normalized box filter.
static Scalar morphologyDefaultBorderValue()
returns "magic" border value for erosion and dilation. It is automatically transformed to Scalar::all...
Definition: imgproc.hpp:1454
Mat getStructuringElement(int shape, Size ksize, Point anchor=Point(-1,-1))
Returns a structuring element of the specified size and shape for morphological operations.
void boxFilter(InputArray src, OutputArray dst, int ddepth, Size ksize, Point anchor=Point(-1,-1), bool normalize=true, int borderType=BORDER_DEFAULT)
Blurs an image using the box filter.
Definition: imgproc.hpp:220
a rectangular structuring element:
Definition: imgproc.hpp:236
double threshold(InputArray src, OutputArray dst, double thresh, double maxval, int type)
Applies a fixed-level threshold to each array element.
Definition: imgproc.hpp:325
void convexHull(InputArray points, OutputArray hull, bool clockwise=false, bool returnPoints=true)
Finds the convex hull of a point set.
double contourArea(InputArray contour, bool oriented=false)
Calculates a contour area.
void resize(InputArray src, OutputArray dst, Size dsize, double fx=0, double fy=0, int interpolation=INTER_LINEAR)
Resizes an image.
Ptr< BackgroundSubtractorMOG2 > createBackgroundSubtractorMOG2(int history=500, double varThreshold=16, bool detectShadows=true)
Creates MOG2 Background Subtractor.
void calcOpticalFlowPyrLK(InputArray prevImg, InputArray nextImg, InputArray prevPts, InputOutputArray nextPts, OutputArray status, OutputArray err, Size winSize=Size(21, 21), int maxLevel=3, TermCriteria criteria=TermCriteria(TermCriteria::COUNT+TermCriteria::EPS, 30, 0.01), int flags=0, double minEigThreshold=1e-4)
Calculates an optical flow for a sparse feature set using the iterative Lucas-Kanade method with pyra...
Definition: nvg.hpp:19
void beginPath()
void lineTo(float x, float y)
void strokeColor(const cv::Scalar &bgra)
void clear(const cv::Scalar &bgra=cv::Scalar(0, 0, 0, 255))
void moveTo(float x, float y)
void strokeWidth(float size)
Definition: framebuffercontext.hpp:25
Source makeCaptureSource(const string &inputFilename)
Sink makeWriterSink(const string &outputFilename, const float fps, const cv::Size &frameSize)