Face Hallucination

Large-Scale L1-Regularized Least Squares for Face Hallucination

Learning the sparse local pixel structure of a face patch from example HR faces

Preprocessing:

Divide each image found by KNN search algorithm for the nearest face images in the training set into a mesh of non-overlapped patches

function [cell_k_patches] = mqSplitImg2Patches(k_images, patch_size, overlap)
global VISUALS;
    %% loop images
    for k = 1 : size(k_images)
        im = k_images{k};
        num_patches = ceil((size(im, 1) * size(im, 2)) / (patch_size * patch_size));
        if(overlap > 0)
            actual_width = size(im, 2) + (size(im, 2) / patch_size)-1;
            num_patches = ceil((size(im, 1) * actual_width) / (patch_size * patch_size));
        end
        patches = cell(num_patches, 1);
        patches_counter = 1;
        %% visualize
        if(VISUALS == true)
            obj_shape_insert = vision.ShapeInserter('BorderColor', 'Custom', 'CustomBorderColor', [46 139 87]);
            visual_im = im;
        end
        %% loop rows
        fin_col = size(im, 2) - patch_size + 1;
        step_col = patch_size - overlap
        for c = 1 : step_col : fin_col
            %% loop cols
            fin_row = size(im, 1) - patch_size + 1;
            for r = 1 : patch_size : fin_row
                patches{patches_counter} = im([r:r+patch_size-1], [c:c+patch_size-1]);
                %% visualize
                if(VISUALS == true)
                    visual_im = step(obj_shape_insert, visual_im, int32([r c patch_size patch_size]));
                end
                %% increment inner most loop counters
                patches_counter = patches_counter + 1;
            end %% end loop columns
        end %% end loop rows
        file_name = sprintf('patches/%d.mat', k);
        save(file_name,'patches');
        if(VISUALS == true)
            figure, imshow(visual_im), title(sprintf('%dth nearest image', k));
        end
    end %% end loop images
end

non-overlapped patches

Inputs

A := Overcomplete Dictionary of Neighbor Pixels
\Psi := Vector of Central Pixels

Overcomplete Dictionary Matrix and Central Pixels Vector

Central Pixels

%% INPUTS:
%%  K := numberf of nearest neighbors
%% OUTPUTS:
%%  CELL OF IPSAY VECTORS OF CENTRAL PIXELS OF ALL PATCHES IN K IMAGES
function [ output_args ] = mqBuildCentralPixels( NUM_NEAREST_NEIGHBORS , PATCH_SIZE)
    %% assume training set is all same size & same # of patches
    k_patches = cell(NUM_NEAREST_NEIGHBORS, 1);
    for k = 1 : NUM_NEAREST_NEIGHBORS
        file_name = sprintf('patches/%d.mat', k);
        k_patches{k} = load(file_name);
        kth_image_patches = k_patches{k}(1);
        k_patches{k} = struct2array(kth_image_patches(1));
    end
    
    %% start building
    num_of_patches = size(k_patches{1}, 1);
    IPSAY = cell(num_of_patches, 1); %% Central Pixels
    CC = ceil(PATCH_SIZE / 2);
    CR = ceil(PATCH_SIZE / 2);
    for j = 1 : num_of_patches
        SINGLE_PATCH_IPSAY = zeros(NUM_NEAREST_NEIGHBORS, 1);
        for i = 1 : NUM_NEAREST_NEIGHBORS
            patch = cell2mat( k_patches{i}(j));
            if (isempty(patch) == false)
                SINGLE_PATCH_IPSAY(i) = patch(CR, CC);
            end
        end
        IPSAY{j} = SINGLE_PATCH_IPSAY;
    end
    save 'central_pixels_for_all_patches.mat' IPSAY;
end

OverComplete Dictionary

%% INPUTS:
%%  K := numberf of nearest neighbors
%% OUTPUTS:
%%  CELL OF OVERCOMPLETE DICTIONARIES OF ALL PATCHES IN K IMAGES
function [ output_args ] = mqBuildOverCompleteDictionary(NUM_NEAREST_NEIGHBORS , PATCH_SIZE)
    %% assume training set is all same size & same # of patches
    k_patches = cell(NUM_NEAREST_NEIGHBORS, 1)
    for k = 1 : NUM_NEAREST_NEIGHBORS
        file_name = sprintf('patches/%d.mat', k);
        k_patches{k} = load(file_name);
        kth_image_patches = k_patches{k}(1);
        k_patches{k} = struct2array(kth_image_patches(1))
    end
    
    %% start building
    num_of_patches = size(k_patches{1}, 1);
    A = cell(num_of_patches, 1); %% Central Pixels
    PSQ = PATCH_SIZE * PATCH_SIZE;
    PSQ_1 = PSQ - 1;
    for j = 1 : num_of_patches
        %% Dictionary for a single Patch
        SINGLE_PATCH_A = zeros(NUM_NEAREST_NEIGHBORS, PSQ_1);
        for i = 1 : NUM_NEAREST_NEIGHBORS
            patch = cell2mat(k_patches{i}(j));
            if (isempty(patch) == false)
                temp = reshape(patch, 1, PSQ);
                len = size(temp, 2);
                cen_left = floor(size(temp, 2) / 2);
                cen_right = ceil(size(temp, 2) / 2) + 1;
                temp_left = temp(1, 1:cen_left);
                temp_right = temp(1, cen_right:len);
                SINGLE_PATCH_A(i, 🙂 = transpose([temp_left temp_right]);
            end
        end
        A{j} = transpose(SINGLE_PATCH_A);
    end
    save 'overcomplete_dictionary_for_all_patches.mat' A
end

Processing

minimization problem

Large-Scale L1-Regularized Least Squares Algorithm

large scale regularized least square

Outputs

\omega := Vector of Coefficients for each central pixel from K HR Training Images

coefficients
reconstructed signal coefficients omega

function [ output_args ] = mqL1LS( input_args )
global VISUALIZE;
VISUALIZE = true;
    ALL_As = load('overcomplete_dictionary_for_all_patches.mat');
    ALL_IPSAYs = load('central_pixels_for_all_patches.mat');
    ALL_As = struct2array(ALL_As);
    ALL_IPSAYs = struct2array(ALL_IPSAYs);
    num_of_patches = size(ALL_As,1);
    coefficients_omega = cell(num_of_patches, 1);
    for p = 1 : num_of_patches
        A = ALL_As(p)
        X0 = ALL_IPSAYs(p)
        if(isempty(A) == false && isempty(X0) == false)
            A = cell2mat(A);
            X0 = cell2mat(X0);
            OMEGA = zeros(size(X0, 1), 1);
            y  = A*X0;          % measurements with no noise
            lambda = 0.01;      % regularization parameter
            rel_tol = 0.01;     % relative target duality gap
            [OMEGA,status]=l1_ls_nonneg(A,y,lambda,rel_tol)
            coefficients_omega{p} = OMEGA;
            if(VISUALIZE == true && strcmp(status, 'Solved'))
                figure(1)
                subplot(2,1,1); bar(X0); ylim([0 255]); title('original signal x0');
                subplot(2,1,2); bar(OMEGA);  ylim([0 255]); title('reconstructed signal x');
            end %% end if solved
        end %% end if patch is not empty
    end %% end loop patches
    save 'coefficients_omega_for_all_patches' coefficients_omega;
end

References:

  1. Face hallucination based on sparse local-pixel structure – Yongchao Lia, Cheng Caia, Guoping Qiub, Kin-Man Lam – ELSEVIER Pattern Recognition, Volume 47, Issue 3, March 2014, Pages 1261–1270, Handwriting Recognition and other PR Applications
  2. L1LS – A Matlab solver for l1-regularized least squares problems – Kwangmoo Koh, Seung-Jean Kim and Stephen Boyd.
  3. http://www.caam.rice.edu/~optimization/L1/FPC_AS/
  4. https://xcorr.net/2014/09/13/off-the-shelf-optimization-functions-in-matlab/
Advertisements

One thought on “Large-Scale L1-Regularized Least Squares for Face Hallucination

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s