Computer Vision · Face Hallucination

Optical Flow Estimation for Face Hallucination

Matlab Implementation

Computer Vision Toolbox exposes more than one optical flow implementations

  1. opticalFlowLK Lucas-Kanade
  2. opticalFlowLKDoG Lucas-Kanade Derivative of Gaussian
  3. opticalFlowHS Horn-Schunk
  4. opticalFlowFarneback Farneback

Test Function

function [ output_args ] = mq_optical_test( input_args )
%MQ_TES Summary of this function goes here
%   Detailed explanation goes here

%% ## BEGIN READ INTERPOLATED LOW RES IMAGE ##
im_path = 'C:\Users\Qamar-ud-Din\Documents\MATLAB\SplitFrame_462.bmp';

lowres = imread(im_path);

figure, imshow(lowres), title('INTERPOLATED LOW RES INPUT IMAGE');
%% ## END READ INTERPOLATED LOW RES IMAGE ##

%% ## BEGIN READ K NEAREST HIGH RES IMAGE ##

%% IMAGE FILE IO
lowres_dir_path = uigetdir;
image_files = dir(strcat(lowres_dir_path, '\*.bmp'));
num_files = length(image_files);

%% LOOP K IMAGES
%% Consider preallocating a variable or array
%% before entering the loop by using zeros, ones, cell, or a similar
%% function
table_of_warping_errors = [];
for i = 1 : num_files
    file_name = image_files(i).name;
    training_image = imread(strcat(lowres_dir_path ,'\', file_name));

    warping_error = mq_opticalflow(lowres, training_image);

    file_name = sprintf('warping_error__%d.txt', i);
    dlmwrite(file_name, warping_error);
    type(file_name)

    table_of_warping_errors = [table_of_warping_errors; warping_error];
end

%% ## END READ K NEAREST HIGH RES IMAGE ##
dlmwrite('warping_errors.dat', table_of_warping_errors);

end

Optical Flow Lucas & Kanade

2016-03-13-111839_1366x768_scrot

function [ output_args ] = mq_opticalflow( interpolated_low_res, training_high_res )
%MQ_OPTICALFLOW Summary of this function goes here
%   Detailed explanation goes here

    %% ## BEGIN VALIDATION ##
    %% check same size as interpolated low res image
    test_width = size(interpolated_low_res,1);
    test_height = size(interpolated_low_res,2);
    training_width = size(training_high_res, 1);
    training_height = size(training_high_res, 2);
    if( test_width ~= training_width || test_height ~= training_height)
        %% throw error
    end

    %% CALCULLATE FLOW
    opticFlow = opticalFlowLK('NoiseThreshold',0.009);
    %% sets previous frame to black %%
    reset(opticFlow);
    %% INTERPOLATED LOW RES IMAGE
    frameGray = rgb2gray(interpolated_low_res);
    estimateFlow(opticFlow, frameGray);

    %% HIGH RES TRAINING IMAGE
    frameGray = rgb2gray(training_high_res);
    flow = estimateFlow(opticFlow, frameGray);

    %% PLOT FLOW
    figure, imshow(frameGray), title('Kth HIGH RES TRAINING IMAGE');
    hold on
    figure, plot(flow,'DecimationFactor',[10 10],'ScaleFactor',10), title('FLOW MAGNITUDE')
    hold off

    %% RETURN MAGNITUDE
    output_args = flow.Magnitude;
end

2016-03-19-130258_1366x768_scrot

Warping Error Calculations

2016-03-19-132719_1366x768_scrot

function [output_args] = mqCalculateWeightsWarpingErrors(Er_k)

    %% USE GLOBALS %%
    global PATCH_WIDTH;
    global PATCH_HEIGHT;
    global EPSILON;
    global BETA;

    %% MATRIX DIMENSION %%
    image_width = size(Er_k, 1);
    image_height = size(Er_k, 2);

    %% HALF PATCH OFFSET %%
    offset_w = floor(PATCH_WIDTH/2);
    offset_h = floor(PATCH_HEIGHT/2);

    %% PADDING %%
    PADDED_Er_k = padarray(Er_k, [offset_w offset_h])

    %% OUTER LOOP PIXELS %%
    b_k = padarray(zeros(image_width, image_height), [offset_w offset_h]);

    %% SET LOOP SENTIMENTS %%
    x_0 = 1 + offset_w;
    x_n = image_width + offset_w;
    y_0 = 1 + offset_h;
    y_n = image_height + offset_h;
    %%%%%%%%%%%%%%%%%%%%%%%%%

    for x = x_0 : x_n
        for y = y_0 : y_n
            %% INNER LOOP PATCH NEIGHBORS %%
            for i = -offset_w : offset_w
                for j = -offset_h : offset_h
                    b_k(x,y) = b_k(x,y) + PADDED_Er_k(x + i, y + j);
                end
            end
            %% END INNER LOOP %%
            b_k(x,y) = power(b_k(x,y) + EPSILON, BETA);
        end
    end

    %%% END OUTER LOOP %%

    %% REMOVE PADDING %%
    output_args = b_k(x_0:x_n, y_0:y_n);
end

Warping Errors Weights Normalization

function [ output_args ] = mq_normalizeWeightsWarpingErrors( input_args )
%UNTITLED Summary of this function goes here
%   Detailed explanation goes here

%%
%%  CONSTANTS SHOULD BE PASSED AS FUNCTION ARGUMENtS
%%  OR MOVE TO FUNCTION mqSetGlobals.m
%%
global PATCH_WIDTH;
global PATCH_HEIGHT;
global EPSILON;
global BETA;
%% mqInitGlobals %%
PATCH_WIDTH = 9;
PATCH_HEIGHT = 9;
EPSILON = 0.009;
BETA = 0.009;
%% END CONSTANTS %%

%% WARPING FILE IO
warping_weights_dir_path = uigetdir;
warping_files = dir(strcat(warping_weights_dir_path, '\*.txt'));
num_files = length(warping_files);

%% LOOP K WARPING FILES
%% Consider preallocating a variable or array
%% before entering the loop by using zeros, ones, cell, or a similar
%% function
b = cell(num_files, 1);
for i = 1 : num_files
    %% FILE READ %%
    file_name = warping_files(i).name;
    b{i} = dlmread(strcat(warping_weights_dir_path ,'\', file_name));
end

%% LOOP SENTIMENTS %%
w = size(b{1}, 1);
h = size(b{1}, 2);

%% B := DIAGONAL WARPING WEIGHTS MATRIX REFER TO PAPER %%
B = cell(w, h, 1);

%% OUTER LOOP ALL PIXELS %%
for x = 1 : w
    for y = 1 : h
        %% INNER LOOP ALL K WARPING WEIGHTS %%
        scaling_denominator = 0;
        for k = 1 : num_files
            scaling_denominator = scaling_denominator + b{k}(x,y);
        end
        %% END INNER LOOP %%

        %% INNER LOOP ALL K WARPING WEIGHTS %%
        diagonal_temp = zeros(num_files, num_files);
        for k = 1 : num_files
            diagonal_temp(k, k) = b{k}(x,y) / scaling_denominator;
        end
        %% END INNER LOOP %%

        B{x,y} = diagonal_temp;
    end
end
%% END OUTER LOOP %%
save B.mat B;
end

2016-03-19-181744_1366x768_scrot.png

opticalFlowHS Horn-Schunk

opticalFlowHS Horn-Schunk

References:

  1. Super-Resolution Optical Flow, Simon Baker and Takeo Kanade, CMU-RI-TR-99-36
  2. High Accuracy Optical Flow Estimation Based on a Theory for Warping – Thomas Brox, Andrés Bruhn, Nils Papenberg, and Joachim Weickert
  3. Is Super-Resolution with Optical Flow Feasible – WenYi Zhao and Harpreet Sawhney
  4. Video Super Resolution using Duality Based TV-L 1 Optical Flow – Dennis Mitzel, Thomas Pock, Thomas Schoenemann, Daniel Cremers
Advertisements

One thought on “Optical Flow Estimation for Face Hallucination

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s