\documentclass[11pt,twoside]{article}
\usepackage{informat}
\usepackage[pdftex]{graphicx}
\usepackage{epstopdf}
\usepackage{epsfig}
\usepackage[colorlinks]{hyperref}
\usepackage{balance}
\usepackage{array}
\usepackage{subfigure}
\usepackage{multirow}
\usepackage[]{algpseudocode}
\usepackage{mathtools}

\newcolumntype{P}[1]{>{\centering\arraybackslash}p{#1}}
\newcolumntype{M}[1]{>{\centering\arraybackslash}m{#1}}

\begin{document}
  \title{Effective Deep Multi-source Multi-task Learning Frameworks for Smile Detection, Emotion Recognition and Gender Classification}
   \author{Dinh Viet Sang\\
      Hanoi University of Science and Technology, 1 Dai Co Viet, Hai Ba Trung, Hanoi, Vietnam\\
      sangdv@soict.hust.edu.vn \\
      \AndAuthor
      Le Tran Bao Cuong\\
       Hanoi University of Science and Technology, 1 Dai Co Viet, Hai Ba Trung, Hanoi, Vietnam\\
      ltbclqd2805@gmail.com}
  \titleodd{Effective Deep Multi-Source Multi-task Learning Frameworks}
  \authoreven{D. Sang et al.}
  \keywords{Multi-task Learning, Convolutional Neural Network, Smile Detection, Emotion Recognition, Gender Classification}
  \received{March 29, 2018}

  \abstract{Automatic human facial recognition has been an active reasearch topic with various potential applications. In this paper, we propose  effective multi-task deep learning frameworks which can jointly learn representations for three tasks: smile detection, emotion recognition and gender classification. In addition, our frameworks can be learned from multiple sources of data with different kinds of task-specific class labels. The extensive experiments show that our	frameworks achieve superior accuracy over recent state-of-the-art methods in all of three tasks on popular benchmarks. We also show that the joint learning helps the tasks with less data considerably benefit from other tasks with richer data.}
  \abstractSi{}

  \maketitle

\section{Introduction}
In recent years, we have witnessed a rapid boom of artificial intelligence (AI) in various fields such as computer vision, speech recognition and natural language processing. A wide range of AI products have boosted labor productivity, improved the quality of human life, and saved human and social resources. Many artificial intelligence applications have reached or even surpassed human levels in some cases.

Automatic human facial recognition has become an active research area that plays a key role in analyzing emotions and human behaviors. In this work, we study different human facial recognition tasks including smile detection, emotion recognition and gender recognition. All of three tasks use facial images as input. In smile detection task, we have to detect if the people appearing in a given image are smiling or not. We then classify their emotions into seven classes: angry, disgust, fear, happy, sad, surprise and neutral in emotion recognition task. Finally, we determine who are males and who are females in gender classification task.

In general, these tasks are often solved as separate problems. This may lead to many difficulties in learning models, especially, when the training data is not large enough. On the other hand, the data of different facial analysis tasks often shares many common characteristics of human faces. Therefore, joint learning from multiple sources of face data can boost the performance of each individual task. 

In this paper, we introduce effective deep convolutional neural networks (CNNs) to simultaneously learn common features for smile detection, emotion recognition and gender classification. Each task takes input data from its corresponding source, but all the tasks share a big part of the networks with many hidden layers. At the end of each network, these tasks are separated into three branches with different task-specific losses. We combine all the losses to form a common network objective function, which allows us to train the networks end-to-end via the back propagation algorithm.

The main contributions of this paper are as follows:
\begin{enumerate}
	\item We propose effective architectures of CNNs that can learn joint representations from different sources of data to simultaneously perform smile detection, emotion recognition and gender classification.
	\item We conduct extensive experiments and achieve new state-of-the-art accuracies in different tasks on popular benchmarks.
\end{enumerate}

The rest of the paper is organized as follows. In section 2, we briefly review related work. In section 3, we present our proposed multi-task deep learning frameworks and describe how to train the networks from multiple data sources. Finally, in section 4, we show the experimental results on popular datasets and compare our proposed frameworks with recent state-of-the-art methods.

\section{Related work}
\subsection{Deep convolutional neural networks}
In recent years, deep learning has been proven to be effective in many fields, and particularly, in computer vision. Deep CNNs are one of the most popular models in the family of deep neural networks. LeNet \cite{lecun1998gradient}, and AlexNet \cite{krizhevsky2012imagenet} are known to be the earliest CNN architectures with not much hidden layers. 

Latest CNNs such as VGG \cite{simonyan2014very}, Inception \cite{szegedy2015going}, ResNet \cite{he2016deep} and DenseNet \cite{huang2016densely} tend to be deeper and deeper. In ResNet, residual blocks can be stacked on top of each other with over 1000 layers. Meanwhile, some other CNN architectures like WideResNet \cite{zagoruyko2016wide} or ResNeXt \cite{xie2016aggregated} tend to be wider. All these effective CNNs have demonstrated their impressive performances in one of the biggest and the most prestigious competitions in computer vision - the annual ImageNet Large Scale Visual Recognition Challenge (ILSVRC). 

\subsection{Smile detection}
Traditional methods often detect smile based on a strong binary classifier with low-level face descriptors. Shan et al. \cite{shan2012smile} propose a simple method that uses the intensity differences between pixels in the gray-scale facial images and then combines them with AdaBoost classifier \cite{viola2002fast} for smile detection. In order to represent faces, Liu et al. \cite{liu2012enhancing} use histograms of oriented gradients (HOG) \cite{deniz2011face}, meanwhile, An et al. \cite{an2015efficient} use local binary pattern (LBP) \cite{ahonen2004face}, local phase quantization (LPQ) \cite{ojansivu2008blur} and HOG. Both of them \cite{liu2012enhancing, an2015efficient} then apply SVM classifier \cite{cortes1995support} to detect smiles. Jain et al. \cite{jain2013smile} propose to use Multi-scale Gaussian Derivatives (MGD) and SVM classifier as well for smile detection.

Some recent methods focus on applying deep neural networks to smile detection. Chen at al. \cite{chen2017smile} use deep CNNs to extract high-level features from facial images and then use SVM or AdaBoost classifiers to detect smiles as a classification task. Zhang et al. \cite{zhang2015facial} introduce two efficient CNN models called CNN-Basic and CNN 2-Loss. The CNN-2Loss is a improved variant of the CNN-Basic, that tries to learn features by using two supervisory signals. The first one is recognition signal that is responsible for the classification task. The second one is expression verification signal, which is effective to reduce the variation of features which are extracted from the images of the same expression class. \cite{sangdv2017smile} propose an effective VGG-like network, called BKNet, to detect smiles. BKNet achieves better results than many other state-of-the-art methods in smile detection.  

\subsection{Emotion recognition}
Classical approaches to facial expression recognition are often based on Facial Action Coding System (FACS) \cite{ekman1997face}. FACS includes a list of Action Units (AUs) that describe various facial muscle movements causing changes in facial appearance. Cootes et al. \cite{van2005model} propose a model based on an approach called the Active Appearance Model \cite{cootes2004statistical} that creates over 500 facial landmarks. Next, the authors apply PCA algorithm to the set of landmarks and derive Action Units (AUs). Finally, a single layered neural network is used to classify facial expressions. 

In Kaggle facial expression recognition competition \cite{kagglecompetition}, the winning team \cite{tang2013deep} propose an effective CNN, which uses the multi-class SVM loss instead of the usual cross-entropy loss. In \cite{sangdv2017emotion}, Sang et al. propose the so-called BKNet architecture for emotion recognition and achieve better performance compared to previous methods.  

\subsection{Gender classification}
Conventional methods for gender classification often take image intensities as input features. \cite{o1997sex} combines the 3D structure of the head with image intensities. \cite{he2004locality} uses image intensities combined with SVM classifier. \cite{baluja2007boosting} tries to use AdaBoost instead of SVM classifier.  \cite{golomb1990sexnet} introduces a neural network trained on a small set of facial images. \cite{ullah2012gender} uses the Webers Local texture Descriptor \cite{chen2010wld} for gender classification. More recently, Levi et al. \cite{levi2015age} present an effective CNN architecture that yields fairly good performance in gender classification.

\subsection{Multi-task learning}
Multi-task learning aims to solve multiple classification tasks at the same time by learning them jointly, while exploiting the  commonalities and differences across the tasks. Recently, Kaiser et al. \cite{kaiser2017one} propose  a big model to learn simultaneously many tasks in nature language processing and computer vision and achieve promising results. Rothe et al. \cite{rothe2015dex} propose a multi-task learning model to jointly learn age and gender classification from images. Zhang et al. \cite{zhang2016joint} propose a cascaded architecture with three stages of carefully designed deep convolutional networks to jointly detect faces and predict landmark locations. Ranjan et al. \cite{ranjan2016hyperface} introduce a multi-task learning framework called hyperface for face detection, landmark localization, pose estimation, and gender recognition. Nevertheless, the hyperface is only trained from a unique source of data with full annotations for all tasks.

\section{Our proposed frameworks}
\subsection{Overall architecture}
In this work, we propose effective deep CNNs that can learn joint representations from multiple data sources to solve different tasks at the same time. The merged dataset (Fig.~\ref{merged}) is fed into a block called ``CNN Shared Network", which can be designed by using an arbitrary CNN architecture such as VGG \cite{simonyan2014very}, ResNet \cite{he2016deep} and so on. The motivation of the CNN Shared Network is to help the networks learn the shared features from multiple datasets across different tasks. It is thought that the features learned in the shared block can generalize better and make more accurate predictions than a single-task model. Moreover, thanks to joint representation learning, the tasks with less data can largely benefit from other tasks with more data. 

After the shared block, each network is separated into three branches associated with three different tasks. Each branch learns task-specific features and has its own loss function corresponding to each task. 

\begin{figure*}[ht!]
	\begin{center}
	\includegraphics[width=0.8\textwidth]{image/merged_data2}
	\end{center}
	\caption{Merged dataset}
	\label{merged}
\end{figure*}

\subsection{Multi-task BKNet}
Our first multi-task deep learning framework called Multi-task BKNet has been previously described in \cite{sang2017multitask} (Fig.~\ref{multitaskBKNet}), which is based on the BKNet architecture \cite{sangdv2017smile, sangdv2017emotion}. We construct the CNN shared network by eliminating three last fully-connected layers of BKNet (Fig.~\ref{BKNet}).

\begin{center}
	\begin{figure}[h!]
		\begin{center}
			\epsfig{file=image2/BKNet1.eps,width=70mm}
		\end{center}
		\caption{The CNN shared network in Multi-task BKNet is just the top part (marked by red lines) of the BKNet architecture \cite{sangdv2017smile}, excluding the last three fully-connected layers.}
		\label{BKNet}
	\end{figure}
\end{center}

\begin{figure}[ht!]
	\begin{center}
	\includegraphics[scale=0.35]{image/BKNet.eps}
	\end{center}
	\caption{Our proposed Multi-task BKNet}
	\label{multitaskBKNet}
\end{figure}
 
\textit{\textbf{CNN Shared Network.}} In this part, we use four convolutional (conv) blocks. The first conv block includes two conv layers with 32 neurons $3 \times 3$ with the stride 1, followed by a max pooling layer $2 \times 2$ with the stride 2. The second conv block includes two conv layers with 64 neurons $3 \times 3$ with the stride 1, followed by a max pooling layer  $2 \times 2$ with the stride 2. The third conv block includes two conv layers with 128 neurons $3 \times 3$ with the stride 1, followed by a max pooling layer $2 \times 2$ with the stride 2. Finally, the last conv block includes three conv layers with 256 neurons $3 \times 3$ with the stride 1, followed by a max pooling layer $2 \times 2$ with the stride 2. Each conv layer is followed by a Batch normalization layer \cite{ioffe2015batch} and a ReLU (Rectified Linear Unit) activation function \cite{nair2010rectified}. The Batch normalization layer reduces the internal covariant shift, and, hence, allows us to use higher learning rate when applying the SGD algorithm to accelerate the training process.

%\begin{center}
%	\begin{figure}[h!]
%		\begin{center}
%			\epsfig{file=image2/VGG2.eps,width=70mm}
%		\end{center}
%		\caption{VGG architecture.}
%		\label{VGG}
%	\end{figure}
%\end{center}


\textbf{\textit{Branch Network.}} After the CNN shared network, we split the network into three branches corresponding to separate tasks, \textit{i.e.}, smile detection, emotion recognition and gender classification. While the CNN shared network can learn joint representations across three tasks from multiple datasets, each branch tries to learn individual features corresponding to each specific task. 

Each branch consists of two fully connected layers with 256 neurons and a final fully connected layer with $C$ neurons, where $C$ is the number of classes in each task ($C = 2$ for smile detection and gender classification branch, and $C = 7$ for emotion recognition branch). Note that, after the last fully connected layer, we can either use an additional softmax layer as a classifier or not, depending on what kind of loss function is being used. These kinds of loss function are described in detail in the next section. Similar with the CNN shared network, each fully connected layer in all branches (except the last one) is followed by a Batch Normalization layer and ReLU. Dropout \cite{srivastava2014dropout} is also utilized in all fully connected layers to reduce overfitting.

\subsection{Multi-task ResNet}

ResNet \cite{he2016deep} is known as one of the most efficient CNN architectures so far. In order to enhance the information flow between layers, ResNet uses shortcut connections between layers. The original variant of ResNet is proposed by He et al. in \cite{he2016deep} with different numbers of hidden layers: ResNet-18, ResNet-34 or ResNet-50, ResNet-101 and ResNet-152. He et al. then introduce an improved variant of ResNet (called ResNet\_v2) in \cite{he2016identity} which shows that the pre-activation order ``conv - batch normalization - ReLU" is consistently better then post-activation order ``batch normalization - ReLU - conv". 

Inspire by the design concept of ResNet\_v2, we propose a multi-task ResNet framework to jointly learn three tasks: smile detection, emotion recognition and gender classification.  Since the amount of facial data is not large, we choose ResNet-50 (with bottleneck layer) as the base architecture to design our multi-task ResNet framework. In the original ResNet\_v2-50 architecture, there are 4 residual blocks, each of which consists of some sub-sampling blocks and identity blocks. The architectures of identity blocks and sub-sampling blocks are shown in Fig.~\ref{Identity_SubSampling}a and Fig.~\ref{Identity_SubSampling}b. For both these two kinds of blocks, we use the bottleneck architecture with \textit{base depth $m$} that consists of three conv layers: a $1 \times 1$ conv layer with $m$ filters followed by a $3 \times 3$ conv layer with $m$ filters and a $1 \times 1$ conv layers with $4m$ filters. The identity blocks and sub-sampling blocks are distinguished by the stride value in the second conv layer and the shortcut connection. In sub-sampling blocks, we use a conv layer with stride 2 instead of stride 1 as in identity blocks. The first residual block of ResNet-50 contains only 3 identity blocks and has no sub-sampling block. The next three residual blocks of ResNet-50 have a sub-sampling block at the top, followed by 3, 5 and 2 identity blocks, respectively.

\begin{figure*} [htb]
	\centering
	\subfigure[Identity block]
	{
		\includegraphics[width=0.4\textwidth]{image/IdentityBlock.eps}}
	\hspace{0.05\textwidth}
	\subfigure[Subsampling block]
	{
		\includegraphics[width=0.4\textwidth]{image/Subsampling.eps}}
	
	%\captionsetup{justification=centering}
	\caption{The architectures of identity blocks and sub-sampling blocks in our Multi-task ResNet framework.}
	\label{Identity_SubSampling}
\end{figure*}

Based on the aforementioned ResNet\_v2-50 architecture, we propose two versions of multi-task ResNet framework. In the first version, which is abbreviated as Multi-task ResNet ver1, we use all of 4 residual blocks to build the CNN shared network to learn joint representations for three tasks. Like in multi-task BKNet, for each task in branch network, we use two fully connected layers with 256 neurons combined with a softmax classifier. Fig.~\ref{ResNet50_FC}a illustrates the architecture of Multi-task ResNet ver1.

In the second version, which is abbreviated as Multi-task ResNet ver2, we only use first three residual blocks to build the CNN shared network. For each task in the branch network, we use a separate residual block combined with  global average pooling layer and a softmax classifier. Fig.~\ref{ResNet50_FC}b illustrates the architecture of Multi-task ResNet ver2.
%\end{mdframed}


\begin{figure*} [ht!]
	\centering
	\subfigure[First version with fully connected layers in the branch network]
	{
		\includegraphics[width=0.4\textwidth]{image/ResNet50_FC}}
	\hspace{0.05\textwidth}
	\subfigure[Second version with residual blocks in the branch network]
	{
		\includegraphics[width=0.45\textwidth]{image/ResNet50}}
	
	%\captionsetup{justification=centering}
	\caption{Our proposed Multi-task ResNet framework. The notation \textit{``Identity block, $m$"} means the identity block with base depth $m$.}
	\label{ResNet50_FC}
\end{figure*}

\subsection{Multi-source Multi-task training}
In this paper, we propose effective deep networks that can learn to perform multi tasks from different data sources. All data sources are mixed together and form a large common training set (Fig.~\ref{merged}). Generally, each sample in the mixing training set is only related to some of the tasks.

Suppose that:
\begin{itemize}
	\item $T$ is the number of tasks ($T=3$ in this paper);
	\item $L_t$ is the individual loss corresponding to the $t^{th}$ task, $t=1,2,..., T$.
	\item $N$ is the number of samples from all training datasets;
	\item $C_t$ is the number of classes corresponding to the $t^{th}$ task ($C_1 = C_3 = 2$ for smile detection and gender classification task, $C_2 = 7$ for emotion recognition task);
	\item $\textbf{s}_i^t$ is the vector of class scores corresponding to $i$-th sample in $t^{th}$ task;
	\item $l_i^t$ is the correct class label of $i$-th sample in $t^{th}$ task;
	\item $\textbf{y}_i^t$ is the one-hot encoding of the correct class label of $i$-th sample in $t^{th}$ task ($y_i^t(l_i^t) = 1$);
	\item $\widehat{\textbf{y}}_i^t$ is the probability distribution over the classes of $i$-th sample in $t^{th}$ task, which can be obtained by applying the softmax function to $\textbf{s}_i^t$.
	\item $\alpha_i^t$ $\in \{0, 1\}$ is the sample type indicator ($\alpha_i^t=1$ if the $i^{th}$ sample is related to the $t^{th}$ task, and $\alpha_i^t=0$ otherwise).
\end{itemize}

Note that, if the $i^{th}$ sample is not related to $t^{th}$ task, then the true label does not exist, and we can ignore $l_i^t$ and $\textbf{y}_i^t$. To ensure the mathematical correctness in this case, we can set them to arbitrary values, for instance, $l_i^t=0$ and $\textbf{y}_i^t$ is a zero vector.  

In this paper, we try two kinds of loss: soft-max cross entropy or multi-class SVM loss. 

The cross-entropy loss requires to use a softmax layer after the last fully-connected layer of each branch. The cross-entropy loss $L_t$ corresponding to $t^{th}$ task is defined as follows:
\begin{align}
L_t = -\dfrac{1}{N}\sum_{i=1}^{N}\left(\alpha_i^t\sum_{j=1}^{C_t}\textbf{y}_i^t(j)log(\widehat{\textbf{y}}_i^t(j))\right),
\label{crossentropyal}
\end{align}
where $\textbf{y}_i^t(j) \in \{0,1\} $ indicates whether $j$ is the correct label of $i$-th sample; $\widehat{\textbf{y}}_i^t(j) \in [0,1] $ expresses the probability that $j$ is the correct label of $i$-th sample. 

The multi-class SVM loss function is used when the last fully connected layer in each task-specific branch accompanies with no activation function. The multi-class SVM loss function corresponding to the $t^{th}$ task can be defined as follows:
\begin{align}
L_t = \dfrac{1}{N}\sum_{i=1}^{N}\left(\alpha_i^t\sum_{\substack{j=1 \\ j\neq l_i^t }}^{C_t}max(0,\textbf{s}_i^t(j)-\textbf{s}_i^t(l_i^t)+1)^2\right),
\label{l2svm}
\end{align}
where $\textbf{s}_i^t(j) $ indicates the score of class $j$ in the $i$-th sample; $\textbf{s}_i^t(l_i^t)$ defines the score of true label $l_i^t$ in the $i$-th sample.

The total loss of the network is computed as the weighted sum of the three individual losses. In addition, we also add L2 weight decay term associated with all network weights $\textbf{W}$ to the total network loss to reduce overfitting. The overall loss can be defined as follows:
\begin{equation}
L_{total}=\sum_{1}^{T} \mu_t L_t + \lambda \|\textbf{W}\|_2^2,
\end{equation}
where $\mu_t$ is the importance level of the $t^{th}$ task in the overall loss; $\lambda$ is the weight decay coefficient.   

We train the network end-to-end via the standard back propagation algorithm.


\subsection{Data pre-processing}
All the images from the datasets that we use later are portraits. Nevertheless, our networks works with facial regions only. Thus, we have to perform data pre-processing to crop faces from the original images in the datasets. 
Here we use Multi-task Cascaded al Networks (MTCNN) \cite{zhang2016joint} to detect faces in each image. Fig.~\ref{MTCNN} shows some examples of using MTCNN for cropping faces.

After that, the cropped images are converted to grayscale and resized to $48 \times 48$ ones.

\begin{center}
	\begin{figure}[h!]
		\begin{center}
			\epsfig{file=image2/MTCNN.eps,width=0.4\textwidth}
		\end{center}
		\caption{MTCNN for face detection. The top row is original images. The bottom row are cropped faces using MTCNN.}
		\label{MTCNN}
	\end{figure}
\end{center}

\subsection{Data augmentation}
Due to small amount of samples in the dataset, we use data augmentation techniques to generate more new data for the training phase. These techniques help us to reduce overfitting and, hence, to learn more robust networks.

We used three following popular ways for data augmentation: 

- Randomly crop: We add margins to each image in the datasets and then crop a random area of that image with the same size as the original image;

- Randomly flip an image from left to right;

- Randomly rotate an image by a random angle from $-15^\circ$ to $15^\circ$. The space around the rotated image is then filled with black color.

In practice, we find that applying augmentation techniques greatly improves the performance of the model.

\section{Experiments and evaluation}
\subsection{Datasets}
\subsubsection{GENKI-4K dataset}
GENKI-4K is a well-known dataset used in smile detection task. This dataset includes 4000 labelled images of human face from different ages, and races. Among these pictures, 2162 images were labeled as smile and 1838 images were
labeled as non-smile. The images in this dataset are taken from the internet with different real-world contexts (unlike other face data sets, often taken in the same scene), which makes the detection more challenging. However, some images in the dataset are unclear (not sure whether smile or not). In some previous works, some unclear images are eliminated during the training and testing phases. It is obviously that keeping wrong samples in the dataset intuitively makes the model more likely to be confused during the training phase. In the testing phase, the wrong samples might considerably reduce the overall accuracy, when the model makes true predictions but the data says no. Despite that fact, in this work we still retain all the images in the original dataset in both phases. Fig.~\ref{genki_ex} shows some examples from GENKI-4K dataset.

\begin{center}
	\begin{figure}[h!]
		\begin{center}
			\epsfig{file=image2/genkiexample.eps,width=0.4\textwidth}
		\end{center}
		\caption{Some samples in the GENKI-4K dataset. The top two rows are examples of smile faces and the bottom two rows are examples of non-smile faces.}
		\label{genki_ex}
	\end{figure}
\end{center}

\subsubsection{FERC-2013 dataset}
FERC-2013 dataset is provided on the Kaggle facial expression competition. The dataset consists of 35,887 gray images of 48x48 resolution. Kaggle has divided into 28,709 training images, 3589 public test images and 3589 private test images. Each image contains a human face that is not posed (in the wild). Each image is labeled by one of seven emotions: angry, disgust,
fear, happy, sad, surprise and neutral. Some images of the FERC-2013 dataset are showed in Fig.~\ref{fer_ex}.

\begin{center}
	\begin{figure}[h!]
		\begin{center}
			\epsfig{file=image2/fer.eps,width=0.4\textwidth}
		\end{center}
		\caption{Some samples in the FERC-2013 dataset.}
		\label{fer_ex}
	\end{figure}
\end{center}

\subsubsection{IMDB and Wiki dataset}
In this work, we use IMDB and Wiki datasets as data sources for gender classification task.

The IMDB dataset is a large face dataset that includes data from celebrities. The authors take the list of the most popular 100,000 actors as listed on the IMDB website and (automatically) crawl from their profiles date of birth, name, gender and all images related to that person. The IMDB dataset contains about 470.000 images. In this paper, we only use $170.000$ images from IMBD. The Wiki dataset also includes data from celebrities, which are crawled data from Wikipedia. The Wiki dataset contains about 62.000 images and in this work we will use about 34.000 images from this dataset. Fig.~\ref{wiki_ex} shows some samples from IMDB and Wiki datasets.

\begin{center}
	\begin{figure}[h!]
		\begin{center}
			\epsfig{file=image2/wiki_sample.eps,width=0.4\textwidth}
		\end{center}
		\caption{Some samples in the IMDB and Wiki datasets.}
		\label{wiki_ex}
	\end{figure}
\end{center}

\subsection{Implementation detail}
In the experiments, we use GENKI-4K dataset for smile detection, FERC-2013 for emotion recognition. We separately use one of the two IMDB and Wiki datasets for gender classification task. %(we call it as gender dataset)

Our experiments are conducted using Python programing-language on computers with the following specifications: Intel Xeon E5-2650 v2 Eight-Core Processor 2.6GHz 8.0GT/s 20MB, Ubuntu Operating System 14.04 64 bit, 32GB RAM, GPU NVIDIA TITAN X 12GB.

\textbf{Preparing data set:} Firstly, we merge three datasets (GENKI-4K, FERC-2013, gender dataset IMDB/Wiki) to make a large dataset. We then create a marker vector to define sample type indicators $\alpha_i^t$. We always keep the number of training data for each task equally to help the learning process stability. For example, if we train our model with two dataset: dataset A with 3000 samples, dataset B with 30000 samples, we will duplicate dataset A $10$ times to make a big dataset with total $60000$ samples. 

In our work, we divide each dataset into training set and testing set. With GENKI-4K dataset, we use 3000 samples for training and 1000 samples for testing. With FERC-2013 dataset we use data split as provided by Kaggle. With Wiki dataset, we use 30000 samples for training and about 4200 samples for testing. With IMDB dataset, we use 150000 samples for training and about 20000 samples for testing.

\textbf{Training phase:} 
With Multi-task BKNet architecture, our model is trained end-to-end by using SGD algorithm with momentum 0.9. We set the batch size equal to 128. We initialize all weights using a Gaussian distribution with zero mean and standard deviation 0.01. The L2 weight decay is $\lambda = 0.01$. All the tasks have the same importance level $\mu_1=\mu_2=\mu_3=1$. The dropout rate for all fully connected layers is set to 0.5. Moreover, we apply an exponential decay function to decay the learning rate through time. The learning rate at step $k$ is calculated as follows:

\begin{align}
curLr = initLr * decayRate ^ {m/decayStep},
\end{align}
where $curLr$ is the learning rate at step $m$; $initLr$ is the initialization learning rate at the beginning of training phase; $decayStep$ is the number of steps when the learning rate decayed.

In our experiment, we set $initLr = 0.01$, $decayRate = 0.8$ and $decayStep = 10000$. We train our Multi-task BKNet model in 250 epochs.

Similar to Multi-task BKNet, we train our Multi-task ResNet end-to-end by using SGD algorithm with momentum $0.9$. We set the batch size equal to 128. We initialize all weights using variance scaling initializer (He initializer). The L2 weight decay is $10^{-4}$. All the tasks have the same important level $\mu_1=\mu_2=\mu_3=1$. We train the Multi-task ResNet ver1 in 100 epochs and train the Multi-task ResNet ver2 in 80 epochs. The initial learning rate is $0.05$ and then decreased by 10 times whenever the training loss stops improving.

\textbf{Testing phase:}
In the testing phase, our model is evaluated by $k$-fold cross-validation algorithm. This method splits our original data into $k$ parts of the same size. The model evaluation is performed through loops, each loop selects $k-1$ parts of data as training data and the rest is used for testing model. For the convenience of doing comparison between different methods, we use $4$-fold cross-validation algorithm as previous works. We will report the average accuracy and the standard deviation after 4 iterations. Moreover, we test our model with two different loss functions mentioned above.

Furthermore, we ensemble different checkpoints obtained during the training phases to infer test samples. In the paper, we keep 10 last checkpoints corresponding to 10 last training epochs for inference.

\subsection{Experimental result}
\subsubsection{Multi-task BKNet}
In this work, we set up two experiment cases. The first one, we train our model with GENKI-4K, FERC-2013 and Wiki dataset. The second one, we train our model with GENKI-4K, FERC-2013 and IMDB dataset. Table~\ref{setup1} shows our experiment setup.

We report our results and compare with previous methods in Table~\ref{result1}. As we can see, using cross-entropy loss function gives better result than using SVM loss function in all cases. 

In smile detection task, the best accuracy we achieve is ${96.23 \pm 0.58 \%}$ when we train our model with GENKI-4K, FERC-2013 and IMDB dataset. In all experiment cases, we achieve better results than previous state-of-the-art methods. Especially, the Multi-task BKNet clearly outperforms the single-task BKNet \cite{sangdv2017smile}. This fact proves that the smile detection task largely benefits from other tasks thanks to sharing the commonalities between data.  

In emotion recognition task, the best accuracy we achieve is ${71.03 \pm 0.11 \%}$ for public test and ${72.18 \pm 0.23 \%}$ for private test. This result  considerably outperforms all of previous methods. 

In gender classification task, to the best of our knowledge, there are no previous results on the Wiki and IMDB datasets for gender classification. In this paper, we apply the single-task BKNet model \cite{sangdv2017smile} and achieve the accuracy $95.82 \pm 0.44 \%$ and $91.17 \pm 0.27 \%$ on the Wiki and IMDB datasets, respectively. The best accuracy we get on Wiki is ${96.33 \pm 0.16 \%}$ when we train our Multi-task BKNet model on Wiki. The best accuracy we get on IMDB is ${92.20 \pm 0.11 \%}$ when we train our model on IMDB. We also report the test accuracy on IMDB when we train the model on Wiki, and the test accuracy on Wiki when we train the model on IMDB.

In all tasks, the Multi-task BKNet yields comparative results and even better than the single-task BKNet in many cases. Furthermore, it should be emphasized that the Multi-task network can effectively solve all the three tasks by using only a common network instead of three separate ones, which would requires approximately three times more memory storage and computational complexity.   

\begin{table*}
	\caption{Experiment setup}
	\centering
	\begin{tabular}{ |c | c| c| c|}
%	\begin{tabular}{ | M{1.8cm} | M{1.8cm}| M{2.2cm}| M{1.4cm}|}
		\hline
		Name & Datasets & Loss function & Use ensemble? \\
		\hline
		\hline
		Config A1 & GENKI-4K, FERC-2013, IMDB & SVM loss & No \\
		\hline
		Config A2 & GENKI-4K, FERC-2013, IMDB & Cross-entropy loss & No \\
		\hline
		Config A3 & GENKI-4K, FERC-2013, IMDB & SVM loss & Yes \\
		\hline
		Config A4 & GENKI-4K, FERC-2013, IMDB & Cross-entropy loss & Yes  \\
		\hline
		Config B1 & GENKI-4K, FERC-2013, Wiki & SVM loss & No \\
		\hline
		Config B2 & GENKI-4K, FERC-2013, Wiki & Cross-entropy loss & No \\
		\hline
		Config B3 & GENKI-4K, FERC-2013, Wiki & SVM loss & Yes \\
		\hline
		Config B4 & GENKI-4K, FERC-2013, Wiki & Cross-entropy loss & Yes  \\
		
		\hline
	\end{tabular}
	
	\label{setup1}
\end{table*}

\begin{center}
	\begin{figure}[h!]
		\begin{center}
			\epsfig{file=image2/false.eps,width=0.4\textwidth}
		\end{center}
		\caption{Some samples that our Multi-task BKNet gives wrong predictions.}
		\label{false}
	\end{figure}
\end{center}


\begin{table*}[ht!]
	\caption{Accuracy comparison on four datasets}
	\centering
	\begin{tabular}[htp]{ | M{6cm} | M{1.8cm}| M{1.8cm} | M{1.8cm} | M{1.8cm} | M{1.8cm} |} 
		\hline
		
		\multirow{2}{*}{Method} & \multirow{2}{*}{GENKI-4K} & \multicolumn{2}{M{3.6cm} |}{FERC-2013} & \multirow{2}{*}{Wiki} & \multirow{2}{*}{IMDB}\\
		\cline{3-4}
		& & Public test & Private test & &\\
		\hline
		\hline
		Chen et al \cite{chen2017smile} & $91.8 \pm 0.95 $ & - & - & - & -\\
		\hline
		CNN Basic \cite{zhang2015facial} & $93.6 \pm 0.47 $ & - & - & - & -\\
		\hline
		CNN 2-Loss \cite{zhang2015facial} & $94.6 \pm 0.29 $ & - & - & - & -\\
		\hline
		Single-task BKNet + Softmax \cite{sangdv2017smile} & $95.08 \pm 0.29 $ & - & - & $95.82 \pm 0.44$* & $91.16 \pm 0.27$*\\
		\hline
		CNN (team Maxim Milakov - rank 3 Kaggle)& - &  $68.2$ & $68.8 $& - & -\\
		\hline
		CNN (team Unsupervised - rank 2 Kaggle)& - & $69.1$ & $69.3 $& - & -\\	
		\hline
		CNN+SVM Loss ({\em team RBM}) \cite{tang2013deep} & - & $69.4 $ & $71.2 $ & - & -\\
		\hline
		Single-task BKNet + SVM loss \cite{sangdv2017emotion} & - & $71.0 $ & $71.9 $ & - & -\\
		\hline
		\hline
		Our Multi-task BKNet (Config A1) & $95.25 \pm 0.43 $ & $68.10 \pm 0.14 $ & $69.10 \pm 0.57 $ & $93.33 \pm 0.19 $ & $89.60 \pm 0.22 $\\ 
		\hline
		Our Multi-task BKNet (Config A2) & $95.56 \pm 0.66 $ & $68.47 \pm 0.33 $ & $69.40 \pm 0.21 $ & $93.67 \pm 0.26 $ & $90.50 \pm 0.24 $\\ 
		\hline
		Our Multi-task BKNet (Config A3) & $95.60 \pm 0.41 $ & $70.43 \pm 0.19 $ & $71.90 \pm 0.36 $ & $93.70 \pm 0.37 $ & $91.33 \pm 0.42 $\\ 
		\hline
		Our Multi-task BKNet (Config A4) & $\textbf{96.23} \pm \textbf{0.58 }$ & $70.15 \pm 0.19 $ & $71.62 \pm 0.39 $ & $94.00 \pm 0.24 $ & $\textbf{92.20} \pm \textbf{0.11 }$\\
		\hline
		Our Multi-task BKNet (Config B1)& $95.25 \pm 0.44 $ & $68.60 \pm 0.27 $ & $69.28 \pm 0.41 $ & $95.25 \pm 0.15 $ & $88.18 \pm 0.26 $\\ 
		\hline
		Our Multi-task BKNet (Config B2) & $95.13 \pm 0.20 $ & $69.12 \pm 0.18 $ & $69.40 \pm 0.22 $ & $95.75 \pm 0.18 $ & $88.68 \pm 0.15 $\\ 
		\hline
		Our Multi-task BKNet (Config B3) & $95.52 \pm 0.37 $ & $70.63 \pm 0.11 $ & $71.78 \pm 0.08 $ & $95.95 \pm 0.15 $ & $88.83 \pm 0.18 $\\ 
		\hline
		Our Multi-task BKNet (Config B4) & $95.70 \pm 0.25 $ & $\textbf{71.03} \pm \textbf{0.11 }$ & $\textbf{72.18} \pm \textbf{0.23 }$ & $\textbf{96.33} \pm \textbf{0.16 }$ & $89.34 \pm 0.15 $\\ 
		\hline
		\hline
		Our Multi-task ResNet ver1 (Config B4)& $95.55 \pm 0.28 $ & $70.09 \pm 0.13 $ & $71.55 \pm 0.19 $ & $96.03 \pm 0.22 $ & $89.01 \pm 0.18 $\\ 
		\hline
		Our Multi-task ResNet ver2 (Config B4)& $95.30 \pm 0.34 $ & $69.33 \pm 0.31 $ & $71.27 \pm 0.11 $ & $95.99 \pm 0.14 $ & $88.88 \pm 0.07 $\\ 
		\hline
	\end{tabular}
	
	\label{result1}
\end{table*}

\subsubsection{Multi-task ResNet}
Based on the experimental results of Multi-task BKNet, we will choose the best config B4 in Table~\ref{setup1} to evaluate our Multi-task ResNet framework.

The results of our Multi-task ResNet are also shown in Table~\ref{result1}. As one can see, our first version yield better results than the second version in all three tasks. 

In smile detection task, the first version of multi-task ResNet achieves ${95.55 \pm 0.28 \%}$ accuracy, while the second version achieves  ${95.30 \pm 0.34 \%}$ accuracy. With the same config B4, our Multi-task BKNet model achieves ${95.70 \pm 0.25 \%}$ accuracy, which is slightly better then Multi-task ResNet.

In emotion recognition task, the accuracy of the first version of Multi-task ResNet is ${70.09 \pm 0.13 \%}$ for public test set and ${71.55 \pm 0.19 \%}$ for private test set. The accuracy of the second version is a little bit lower with  ${69.33 \pm 0.31 \%}$ and ${71.27 \pm 0.11 \%}$ for public test set and private test set, respectively. In this task, both versions of Multi-task ResNet seem to clearly lose Multi-task BKNet, which obtains higher approximately 1\% accuracy in each test set.

In gender classification task, both our variants of multi-task ResNet yield pretty good results, which compete with the results of of the multi-task BKNet model. The first variant achieves the accuracy of ${96.03 \pm 0.22 \%}$ and ${89.01 \pm 0.18 \%}$ for Wiki dataset and IMDB dataset, respectively. The second variant achieves the accuracy of ${95.99 \pm 0.14 \%}$ for Wiki dataset and ${88.88 \pm 0.07 \%}$ for IMDB dataset. 

The experiment results show that the Multi-task ResNet is slightly worse than the Multi-task BKNet in all tasks. The reason could be due to that ResNet with a pretty deep architecture and fairly large number of parameters tends to be over-complex w.r.t the mixing training data across the three tasks and leads to overfitting. Meanwhile, BKNet is quite smaller than ResNet, and is capable to fit the data  better. 

\subsubsection{Speed performance comparison between different frameworks}

In Table~\ref{time_inference} and Table~\ref{time_training}, we show the inference time and training time of three frameworks: Multi-task BKNet,  Multi-task ResNet ver1 and Multi-task ResNet ver2 with Config B4 (from Table~\ref{setup1}).

As one can see, the Multi-task ResNet ver2 acquires the fastest convergence. Despite a little longer in training time, Multi-task BKNet is significantly faster in inference in comparison with both versions of Multi-task ResNet. The fast inference with high accuracy make the Multi-task BKNet well suitable for real-time applications.

\begin{table}[h!]
	\caption{Comparison of inference time between different frameworks}
	\centering
	\begin{tabular}[htp]{ | M{4.7cm} | M{2.5cm}|} 
		\hline
		Framework & Inference time per image (sec) \\
		\hline
		\hline
		Multi-task BKNet & 0.02  \\
		\hline
		Multi-task ResNet ver1 & 0.065 \\
		\hline
		Multi-task ResNet ver2 & 0.071 \\
		\hline
	\end{tabular}
	\label{time_inference}
\end{table}

\begin{table*}
	\caption{Comparison of training time between different frameworks}
	\centering
	\begin{tabular}[htp]{ | M{6.5cm} | M{3cm}| M{3cm} | M{3cm} |} 
		\hline
		Framework & Number of epochs & Training time per epoch (min) & Total training time (min)\\
		\hline
		\hline
		Multi-task BKNet & 250 & 3.42  & 854 \\
		\hline
		Multi-task ResNet ver1 & 100 & 8.12  & 817  \\
		\hline
		Multi-task ResNet ver2 & 80 & 8.67  & 693  \\
		\hline
	\end{tabular}
	\label{time_training}
\end{table*}


\section{Conclusion}

In this paper, we propose effective multi-souce multi-task deep learning frameworks to jointly learn three facial analysis tasks including smile detection, emotion recognition and gender classification. The extensive experiments in well-known GENKI-4K, FERC-2013, Wiki, IMDB datasets show that our frameworks achieve superior accuracy over recent state-of-the-art methods in all tasks. We also show that the smile detection task with few data largely benefit from the two other tasks with richer data.

In the future, we would like to exploit some new auxiliary
loss function to regulate the model learning process in order
to improve the performance accuracy of neural networks in
various computer vision tasks.

\begin{center}
	\begin{figure}[h!]
		\begin{center}
			\epsfig{file=image2/demo2.eps,width=60mm}
		\end{center}
		\caption{Some results of our Multi-task BKNet framework. The blue box corresponds to females and the red box corresponds to males.}
		\label{demo}
	\end{figure}
\end{center}

\section{Acknowledgments}
This research is funded by Hanoi University of Science and
Technology under grant number T2016-LN-08.

\bibliographystyle{abbrv}

\bibliography{ref} 

\end{document}
