\documentclass[english]{jnlp_3.5}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{hangcaption_jnlp}
%%%%\usepackage{udline}
%%%%\setulminsep{1.2ex}{0.2ex}
%%%%\let\underline=\ul
\allowdisplaybreaks


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
\usepackage{multirow}
\usepackage{tabularx}
%%%%\usepackage{caption, subcaption}
%%%%\usepackage[export]{adjustbox}
\DeclareMathOperator*{\argmax}{arg\,max}
\usepackage{arydshln}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{29}
\Number{1}
\Month{March}
\Year{2022}
%%%%\Issuetitle{Special Issue: XXXXXXXX}

\received{2021}{8}{1}
\revised{2021}{11}{8}
%%%%\rerevised{2006}{5}{6}
\accepted{2021}{11}{29}

\setcounter{page}{166}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Improving Conversation Task with Visual Scene Dataset}
\eauthor{Abdurrisyad Fikri\affiref{mirai} \and Hiep V. Le\affiref{line} \and Takashi Miyazaki\affiref{yahoo} \and Manabu Okumura\affiref{mirai} \and \\
	Nobuyuki Shimizu\affiref{yahoo}} 
\eabstract{%
To build good conversation agents, an accurate conversation context is assumed to be required. We argue that a conversation scene that includes speakers can provide more information on the context because using images as conversation contexts has proven effective. We constructed a visual conversation scene dataset (VCSD) that provided scenic images corresponding to conversations. This dataset provides a combination of (1) conversation scene image (third-person view), (2) the corresponding first utterance and its response, and (3) the corresponding speaker, respondent, and topic object. In our experiments on the response-selection task, we first examined BERT (text only) as a baseline. Although BERT managed to perform well in general conversations, where a response continued from the previous utterance, it failed to deal with cases where visual information was necessary to understand the context. Our error analysis found that conversations requiring visual contexts can be categorized into three types: visual question-answering, image-referring response, and scene understanding. To optimize the usage of conversation scene images and their focused parts, that is, speaker, respondent, and topic object, we proposed a model that received texts and multiple image features as inputs.
Our model can capture this information and achieve 91\% accuracy.
}
\ekeywords{Visual Conversation, Multi-modality, Dataset}

\headauthor{Fikri et al.}
\headtitle{Improving Conversation Task with Visual Scene Dataset}

\affilabel{mirai}{}{Department of Information and Communications Engineering, School of Engineering, Tokyo Institute of Technology}
\affilabel{yahoo}{}{Yahoo Japan Corporation}
\affilabel{line}{}{LINE Fukuoka Corporation}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}
\label{sec:intro}

In real-world conversations, human speakers occasionally need to examine visual clues from the other party to understand the conversation context. 
According to recent studies on multimodal tasks\cite{mostafazadeh-etal-2017-image,visdial,murahari-etal-2019-improving}, adding corresponding images corresponding to the conversation context has proven useful in improving model performance. 
However, because existing studies commonly use topic/object images as visual clues, the relevance of scenic images that include speakers is largely unexplored.
For example, in Figure \ref{fig:example_raw}, because we can view the conversation scene, we can understand why the woman needs to pick the little girl up because the horse is taller than the girl.
In some real-world cases, for example, a conversation agent (robot) sometimes witnesses an interaction between humans, and such a situation requires it to understand conversation from a third-person perspective.
Therefore, we argue that visual scene understanding, that is, using images that include both speakers and surrounding objects, improves visual conversation tasks.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1
\begin{figure}[t]
\begin{center}
\includegraphics{29-1ia7f1.pdf}
\end{center}
Girl: Can I pet it?\newline
Woman: Yes, let me pick you up so you can pet it.
\hangcaption{Woman's response (picking girl up) can be fully understood only by seeing whole scene. Blue boxes marked with S and R denote a \textit{speaker} and a \textit{respondent}, respectively; \textit{Topic object} is marked with yellow box.}
\label{fig:example_raw}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We constructed a new third-person-view visual conversation scene dataset (VCSD) to investigate the impact of visual conversation scenes on conversation tasks.
This proposed dataset provides visual scene images corresponding to conversation utterances; each pair of data consists of (1) an original image, (2) the first utterance and its response, and (3) the corresponding speaker, respondent, and topic object annotations in the conversation image.
Furthermore, compared to the commonly used minimum setting, that is, using only conversation sentence pairs and a corresponding topic image, our dataset setting provides the additional annotations of the speaker, respondent, and topic object. Using our proposed setting, we were able to explore more combinations between the original scenic image and additional focused part(s) of the image.
We obtained the original images from the Visual Genome\footnote{\url{https://visualgenome.org}}\cite{krishnavisualgenome} and selected speakers/respondents/topic objects from their annotations.
We then annotated the images with conversation utterances using a crowdsourcing service.
Using images from Visual Genome, our dataset can be used mutually with Visual Genome's dense annotations.

With our proposed dataset, we investigated and analyzed the effectiveness of using visual scenes in conversation settings.
We experimented with both single and multimodal models in a binary classification setting to predict whether the response was correct.
To confirm our hypothesis that visual clues positively improve conversation task performance, we first experimented with a BERT \cite{devlin-etal-2019-bert} (text-only) model as a baseline.
From BERT's outputs, we found that approximately 15\% of the outputs demonstrated the importance of visual information in capturing the conversation context.
In addition, in a conversation task, responding to an utterance correctly and capturing the conversation context can be independent if the conversation does not belong to the following three types of visual conversation, which are determined by us: (1) visual question-answering, (2) image-referring response, and (3) scene understanding. These types are discussed in Section \ref{sec:experiment}.
We also proposed a model to achieve a better understanding of the multimodal contexts, comprising texts, scenic images, and their corresponding focused parts.
Our proposed model performs better than the baselines and improves the performance of these three types.

In summary, this study offers a three-fold contribution. (1) We introduced a new multimodal conversation dataset.\footnote{The proposed dataset (VCSD) will be made available online.}
To our knowledge, our third-person-view dataset is one of the first attempts to address the impact of visual conversation scenes in conversation tasks.
We selected 8,336 conversation images and annotated them, resulting in 22,331 pairs of image-conversation utterances. This dataset can also be used with the Visual Genome's dense image annotations.
(2) We investigated and analyzed the effectiveness of using a visual scene and its components to improve conversation task performance.
Furthermore, we provided a neural network model that can utilize visual scene information better than baselines.
Our results show that using visual scene images, particularly utilizing speakers and object parts, improves performance.
(3) We propose new visual conversation types that can help navigate future work on cross-modal conversation tasks.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Work}
\label{sec:related_work}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Image-based Conversation Datasets}

There are several situations in which conversations are coupled with visual information, and many tasks and datasets have been proposed to deal with these situations effectively using intelligent systems or robots.
In these situations, the relationship between a speaker and an image can be classified into three cases: the speaker cannot see the image, the speaker can see the image, and the speaker is inside the image.

Visual dialogue  \cite{visdial} and GuessWhat?! \cite{Vries_2017_CVPR} can be classified into a task setting where a person who cannot see an image repeatedly asks questions about the image to the person who can see it. These tasks can be seen as visual question answering  \cite{VQA} with multiple turns.

In the IGC \cite{mostafazadeh-etal-2017-image} and Image-Chat \cite{shuster-etal-2020-image} datasets, both speakers can see the image and talk about their impressions of the image. 
\citeA{10.1145/3173574.3173851} collected one million conversations associated with images from Twitter. They analyzed the influence of scene sentiment and facial expression features on conversation response generation tasks.
The MMD dataset \cite{DBLP:conf/aaai/SahaKS18} simulates a process in which a shopper tries to find a product to buy with the help of an agent. In the dialogue, the shopper and agent can see multiple product images from which to select.
PhotoBook \cite{haber-etal-2019-photobook} is a dataset for studying how people achieve common ground in a dialogue using a multi-round online image-identification game online.

VFD \cite{kamezawa-etal-2020-visually} assumes a ``first-person view'' setting where an agent (a person or a robot) sees a visual scene provided as an image and talks to a person within the image.

Our dataset differs from the previous studies above in that it considers a situation in which two people having a conversation exist in the images.
This setting is useful for developing conversation agents that can understand a conversation in a visual context from a third-person perspective.
Table \ref{tab:dataset_comparison} summarizes the characteristic properties of the datasets described in this section. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[t]
\input{07table01.tex}
\caption{Existing dialogue datasets associated with visual information.}
\label{tab:dataset_comparison}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Vision and Language Models}

With the advancement in deep learning, the performance of image recognition and language processing has been greatly improved, and methods for integrating images and texts have been actively researched.
Since the success of BERT \cite{devlin-etal-2019-bert}, pre-trained vision and language models have rapidly become popular owing to their improved performance in a wide variety of vision and language tasks.
Pre-trained vision and language models can be classified into two architecture types.
In single-stream architecture models such as UNITER \cite{chen2020uniter}, a single transformer module takes both visual and textual inputs.
In two-stream architecture models such as LXMERT \cite{tan-bansal-2019-lxmert} and ViLBERT \cite{lu2019vilbert}, one transformer module takes textual inputs, and the other module uses visual inputs.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Dataset}

As described in Section \ref{sec:related_work}, several datasets covering visual and language tasks are publicly available. However, because we intend to investigate the effectiveness of visual conversation images or the conditions required for this kind of image, we concluded that the existing datasets are not suitable. Therefore, to accommodate this intention, we constructed our own dataset.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Scenario Description}
\label{ssec:scenario}

In a real-world situation, a conversation agent (robot) sometimes witnesses an interaction between humans in front of it, so the agent should be capable of understanding the conversation from a third-person perspective.
This agent needs to determine who is conversing and what they are talking about by evaluating their utterances and visual clues. 
Figure \ref{fig:example_raw} illustrates a scene captured by the agent.
Given the first utterance and the visual clues, we task the agent by predicting the correct response from available candidates, that is, we implement this task as a multimodal conversation response-selection task.

The minimal inputs for the robot are:  
(1) Original image (a conversation scene), 
(2) an utterance made by a speaker, and 
(3) response candidate. 
As a novel feature of our dataset, we can choose to input (4) a speaker or (5) a respondent from the conversation scene.
We consider specifying the speaker and the respondent to be a natural setting for robots because it should be clear who made an utterance and who should respond to it.
Another interesting possible input to the robot, though not realistic, is (6) a topic object.
This setting is where the robot has prior knowledge of the content of the conversation.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Dataset Construction}
\label{ssec:dataset-constrcution}

We constructed our dataset by annotating the images from the Visual Genome dataset \linebreak
\cite{krishnavisualgenome} with three stages of crowdsourcing.
We defined an image as a conversation scene if it satisfied three criteria: (1) having two or more persons, (2) the speakers are relatively close to each other, and (3) there is at least one nearby object that can be used as a general conversation topic. 
We used Visual Genome annotations such as ``person,'' ``man,'' ``woman,'' and ``kid'' for speakers and respondents, while topic objects could be any object, e.g., building, sky, or hat.
We call this set of images \textit{potential images}. The obtained 16,333 \textit{potential images} were then annotated by crowd workers via Amazon Mechanical Turk. The number of workers who participated in this study was 1539. We specified English-speaking countries and native fluency as the minimum requirements.
Three stages of crowdsourcing were done by different groups of workers.

In the first stage of crowdsourcing, we filtered out images that were unsuitable for conversation scenes. 
Using Visual Genome annotations, for each image, we marked two speaker/respondent objects with bounding boxes. Please note that one image can contain several ``person'' objects; hence, one image can have multiple potential speaker/respondent pairs. Then, we asked the crowd workers to determine whether the pair of speakers appeared to be talking. The crowd workers needed to choose among five answers: \textit{yes, likely, possibly, no} and \textit{invalid (no visible two speakers)}. We retained only the images labeled with \textit{yes} and \textit{likely}, amounting to 8,356 images.

In the second stage, 
We asked the crowd workers to choose between one of the two annotated speakers as the first speaker, and then select one object from the image as the general topic of conversation. 
Afterward, the crowd workers were required to pose as the first speaker and write an appropriate first utterance. 
To increase the conversation diversity, we annotated each image four times. Therefore, the same image could have different first speakers or topics. We also asked crowd workers to label a sentence as natural or not. Later, we dismissed the data labeled as unnatural.

The last annotation stage involved collecting responses appropriate to the first speaker's utterances. The crowd workers were required to pose as the second speaker and write a relevant response to the first utterance. While we asked the annotators to consider the topic object, the response did not need to explicitly contain the topic word. The response implicitly refers to the topic object. For example, if the topic object was ``a building,'' it was possible to respond with ``do you want to go there?''
Annotators could also include a comment if the topic object was unclear, for example, blurry, or if the image quality was low. We removed the data that they regarded as unclear.
By the end of this stage, we collected 22,331 pairs of \textit{image-first utterance-response} data.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Dataset Analysis}
\label{ssec:dataset-analysis}

To investigate whether visual scenes are necessary to improve conversation tasks, we performed a qualitative analysis of the utterance and response pairs.
We sampled 100 pairs from our dataset and evaluated them, especially the utterance, with two questions: (1) ``Is the image necessary to understand the context?'', and (2) ``Do we need context understanding to respond to the utterance?'' In this analysis, we hid the corresponding images from the assessors. Our findings show that 75\% of conversations need to be accompanied by visual information to be correctly understood. 
However, only 58\% of conversation utterances required context understanding to respond to. For example, with the ``Let's pick up the cones'' utterance, we need to see the visual clue to understand the context, but this utterance can be responded to with a simple ``Yes'' or ``Sure, no problem.''
This phenomenon indicates that the ability to respond does not necessarily correspond to context understanding.

We also performed a cluster analysis on the conversation text to understand the distribution of conversation topics.
Using GloVe embedding \cite{pennington2014glove}, we converted words to 300-dimensional vectors and then averaged the word embeddings to obtain a vector representation of every pair in conversations. Then, we used agglomerative clustering \cite{Rokach2005,scikit-learn} to obtain 60 clusters of conversations.
The clusters showed that our dataset has a diverse set of conversation topics. 
We selected four clusters, reduced the vector dimensions using principal component analysis (PCA), and visualized them in Figure \ref{fig:dataset_cluster}. From these four clusters, that is, \textit{food}, \textit{fashion}, \textit{sport}, and \textit{animal}, we can observe that a conversation from one cluster is sometimes similar to that in another cluster. For instance, conversations on the topic of \textit{animal} might be related to those on the topic of \textit{sport}, and vice versa. 
We also found that the conversation topics in our dataset cover topics indicated not only by ``nouns'' but also ``verbs,'' e.g., \textit{travel} or \textit{transportation}, that can be indicated by the words ``go,'' ``come,'' or ``drive.'' 
We can argue that this shows that the focus of natural human conversations is not necessarily about objects, but rather about actions regarding the objects.

We recruited four people from our research group to ensure the quality of the collected data.
Further, we performed a quality assessment of 400 randomly sampled pairs; each person assessed 100 samples. 
We defined four assessment criteria: (1) the relevance of the utterance-response pair, (2) the reasonability of the topic object, (3) the reasonability of the speaker-respondent pair, and (4) the naturalness of the conversation given the visual scene. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 2
\begin{figure}[t]
\begin{center}
\includegraphics{29-1ia7f2.pdf}
\end{center}
\hangcaption{Distribution of conversation topics in VCSD. Each conversation is represented by GloVe vector and colored according to its cluster, i.e., Food, Fashion, Sport, and Animal.}
\label{fig:dataset_cluster}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 2
\begin{table}[t]
\input{07table02.tex}
\hangcaption{Quality assurance for image-conversation in terms of utterance-response \textit{relevance}, \textit{object} and \textit{speakers} appropriateness, and \textit{naturalness}.} 
\label{tab:quality_assurance}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The assessors were required to assign an assessment score on a 4-point scale, with 1 being the lowest score (clearly wrong), 2 (inappropriate), 3 (questionable), and 4 as the highest score (good). 
From the results shown in Table \ref{tab:quality_assurance}, we concluded that the amount of ``noisy'' data (about 1\% to 3\%) was
acceptable. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experiment}
\label{sec:experiment}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Data}
\label{ssec:experiment-data}

We split our 22,331 pairs of data into training, validation, and test sets at a ratio of 70:15:15; 15,631 for the training set and 3,349 for each the validation and test set. We used these data as the positive instances. 
For negative instances, for each first utterance, we selected a random response from a set of four responses given the same image. 
However, as described in Section \ref{ssec:dataset-constrcution}, we removed pairs of data that were considered unclear or completely wrong. This removal caused some images to have fewer than four response alternatives. Nevertheless, we ensured that the negative samples were selected from conversations using the same original images. In the rare cases when there is no alternative conversation with the same image, we selected other conversations with different images as the negative pairs. 
Some examples can be seen in Figure \ref{fig:visual_conv_type}.
We set the ratio of positive and negative instances to 1:1, resulting in over 30K instances for the training set and over 6K each for the validation and test sets.

Following the scenario described in Section \ref{ssec:scenario}, we needed a method to utilize the focused parts of the images, that is, the speakers and possibly the topic objects.
One approach is to use annotations for image object regions (bounding box positions), as used in Faster-RCNN \cite{NIPS2015_14bfa6bb} for object-detection tasks.
Another approach is to use cropped image parts together with the original images, as used in \cite{ijcai2018-214}. By doing this, the model can learn both representations for an entire image and the focused parts.
Because our proposed dataset uses Visual Genome annotations, we can use either approach to include partial information from images. 
However,the use of cropped images can increase the capacity of visual information and may be useful for model performance.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Transformer-based Baselines and Error Analysis}
\label{ssec:experiment-transformer}

We first experimented with a single-modal (text only) model to confirm our hypothesis that multiple modalities using conversational scene images can improve the conversation task. We used BERT \cite{devlin-etal-2019-bert} as the baseline for this task. We fine-tuned the BERT base model with first utterance--response pairs from our dataset with labels of 1 and 0 for positive and negative samples, respectively.
i.e., as binary classification.
While we expected BERT to achieve a moderate accuracy score 
on this task, according to our analysis in Section \ref{ssec:dataset-analysis}, wherein approximately 50\% of cases would require context understanding, BERT exceeded our expectation by achieving an accuracy score of above 85\%.

Then, we proceeded to experiment with LXMERT \cite{tan-bansal-2019-lxmert} as a BERT counterpart for a cross-modal model. 
For this experiment, we trained LXMERT from scratch and did not use the pre-trained models.
Because LXMERT accepts one image for each corresponding text, we used the original images. 
We trained the model using the same binary-classification task; the model learned to predict whether a pair of utterance-response and a visual scene belong to the same conversation.
Because LXMERT expects the input images to be annotated by bounding boxes, we decided to use two variants of annotations: (1) speaker, respondent, and object bounding boxes, and (2) all Visual Genome's densed-annotation bounding boxes. While both settings produced considerably high scores of 79.4\% and 80.65\%, they were still lower than BERT's achievement.  We postulate that the higher performance of BERT is due to its pretraining that utilizes a large text corpus.  While LXMERT is also based on the BERT architecture, its textual component is not extensively pre-trained compared to BERT.

To better understand these phenomena, we did an error analysis of BERT's outputs
through a manual qualitative observation of randomly sampled outputs. We discovered that,
when there are enough details in the text to assume that a response follows the first utterance, visual clues are not necessary to complete this task. 
For example, in Figure \ref{fig:error_analysis}, BERT successfully predicted the label whereas LXMERT failed to do so. We suspected that LXMERT failed to associate the textual context of ``warm'' and ``sweating'' with the ``cold'' scene. This example shows that, in some cases, the textual context would suffice to respond to an utterance correctly.
Subsequently, from approximately 15\% of cases where BERT failed to predict the correct label, we randomly sampled 100 pairs of false negative cases. 
We then analyzed whether the visual information was required to correctly predict the label. We found out that 23\% of the samples might not require visual clues to be classified correctly, thus resulting in the complete failure of the BERT classification.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 3
\begin{figure}[b]
\begin{center}
\includegraphics{29-1ia7f3.pdf}
\end{center}
\textbf{Utterance}: Are we warm enough?\newline
\textbf{Response}: I'm sweating, you?
\hangcaption{LXMERT failed to predict this pair as \textit{correct}, while BERT and our model correctly predicted it.}
\label{fig:error_analysis}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

For the remaining 77\%, we classified the conversation pairs into three types: (1) visual question-answering type, (2) image-referring response type, and (3) scene-understanding type.
The \textit{visual question-answering} type is quite obvious; it involves cases in which both the speaker and the respondent are mainly asking and answering about the same specific object in an image.
The \textit{image-referring response} type is the one in which the speaker does not explicitly refer to or mention an object, but the respondent does. We can consider this as the opposite of the standard case where the speaker asks or talks about an object, and the respondent only needs to respond accordingly.
Finally, the \textit{scene-understanding} type, is more complicated when setting a boundary to it. As the name implies, this type requires contextual understanding from both the text and its corresponding visual context. It may also require common sense or specific assumptions on the conversation topic. 

Guided by these definitions, we observed that 35\% of the data were of the first type, 11\% were of the second type, and 31\% were of the third type.

From the examples in Figure \ref{fig:visual_conv_type}, we can see that, in the \textit{image-referring response} type, the object ``dog'' in the response text needs to be referred to in the image. While it is similar to the ``scene-understanding'' type, the textual context may seem unrelated to the scene in the ``image-referring response'' one. 
Considering these phenomena, we argue that we need an approach for extracting related and useful information from an image to complement the textual context.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 4
\begin{figure}[b]
\begin{center}
\includegraphics{29-1ia7f4.pdf}
\end{center}
\hfill\begin{minipage}[t]{189pt}
\textbf{Utterance:} I am getting great pictures from the boat. \newline
\textbf{Positive:} You want the dog in some of them?\newline
\textbf{Negative:} Yes, we'll keep riding from here.
\end{minipage}\hspace{6pt}\begin{minipage}[t]{189pt}
\textbf{Utterance:} Are you going to make it to the lift?	\newline
\textbf{Positive:} I'm going to jump out from this snow\newline
\textbf{Negative:} Sure, someday I would love to climb the mountain.
\end{minipage}\hfill
\hangcaption{Examples of visual conversation types: image-referring response (left) and scene-understanding (right).}
\label{fig:visual_conv_type}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Conversation Scene Image Augmented Model}
\label{ssec:experiment-proposed}

To complement the above investigation into the impact of visual scenes on conversation tasks, 
and to confirm whether we can improve the performance of transformer-based models, 
we continued our experiments by using the cropped part(s) of the conversation scene images to aid the conversation text in this task. 
To accommodate the need for feeding multiple images and their pairs of utterance-response corresponding to the model, 
we developed a neural network model with a simple architecture. Figure \ref{fig:main-architecture} shows the architecture of the proposed model.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 5
\begin{figure}[b]
\begin{center}
\includegraphics{29-1ia7f5.pdf}
\end{center}
\hangcaption{(a) Overview of our model architecture being used in experiments. (b) We applied attention over visual features.}
\label{fig:main-architecture}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We adapted the architecture for visual question answering (VQA) tasks by \citeA{DBLP:journals/corr/KazemiE17} and combined it with the strategy of \citeA{ijcai2018-214} for an addressee recognition task to optimize the extraction of visual information for the parts focused on. We modified the objective to predict whether a given pair is appropriate or not, that is, binary classification.
The objective can be formulated as
\begin{equation}
    a^* = \argmax_{a \in [0,1]} p(a|\text{\textbf{x}},\text{\textbf{r}};\theta), 
    \label{eq:objective}
\end{equation}
where \textbf{x} is multimodal input representation including text utterance \textit{u}, original image \textit{i}, and cropped image area \textit{c}, and \textbf{r} is a response candidate. We computed the probability of $a$, the value being between 0 and 1, to obtain the classification $a*$.
To calculate the \(p(a|\text{\textbf{x}},\text{\textbf{r}};\theta)\), we first encode all the inputs and the response using separate neural encoders:
$v_u = f_u(u), v_i = f_i(i), v_c = f_i(c) $ and  $ v_r = f_r(r) $.
We represent the text encoder as \(f_u\) and the image encoder as \(f_i\), respectively, to produce the vectors \(v_u\),\(v_i\), and \(v_c\). We used the same (shared) image encoder for both the original image \(i\) and the cropped image \(c\). Because a cropped image is a part of an original image, we assume it would be more useful for the encoder to be able to learn the mutual information between these visual clues.
In contrast, we used a different text encoder \(f_r\) for response \(r\).
We used a one-layer bidirectional LSTM \cite{Hochreiter:1997:LSM:1246443.1246450} for the text encoder and ResNet \cite{7780459}, pre-trained with the Places356 dataset \cite{zhou2017places}, for the image encoder.

For the original image vector \(v_i\), a soft attention mechanism was used to align the information from the image features and textual context, \(v_u\) and \(v_r\). We adapted the stacked attention used by \citeA{DBLP:journals/corr/KazemiE17} for this mechanism.
This attention mechanism results in vectors \(v_i^u\) and \(v_i^r\), which represent the alignment of the text utterance and response, respectively.
Then, we combined both alignment vectors by element-wise addition: 
\begin{equation}
    v_i^\prime = v_\alpha^u + v_\alpha^r.
    \label{eq:sum_att} 
\end{equation}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Experiment Settings}

We used Adam \cite{kingma2014method} as the optimizer for training and fixed the learning rate at 0.0001. The mini-batch size was set at 20. 
We defined an early stopping condition, as the training convergence indicator, using a parameter called \textit{patience}. We defined \textit{patience} as the maximum number of consecutive iterations allowed when the average training loss is increased to greater than a threshold.
For instance, if we set the \textit{patience} to 10 and the threshold to 5\%, the training should stop if the average loss is increases above 5\% (compared to the best average loss) for ten consecutive iterations.
In our experiments, we fixed the \textit{patience} at five and set the threshold to 5\%.
We trained our models on an Nvidia Titan RTX GPU with a 24-GB GPU-memory capacity. We limited the maximum training epoch to 50 epochs. Typically, the training converged in approximately 12 hours for our model. 
The dimensions of the word embeddings and LSTM were set to 300 and 512, respectively, while the dimension of the image features was set to 2048. The input images consist of one original image and three cropped parts of this original image. 
We pre-processed all these images, the original as well as the cropped ones, to have equal dimensions of $256 \times 256$.  Then, we applied data augmentation to the images during training, including random cropping, horizontal flipping, and normalization transformation.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[b]
\input{07table03.tex}
\hangcaption{Results of our models with different input settings. \textit{Original images} were inputted as \textit{scene images} by default. S, R, and O denote the \textit{speaker}, \textit{respondent}, and \textit{topic object} cropped images, respectively.  
Difference between a pair of * marked scores was not statistically significant.
The differences between all other pairs (from our model) were significant (t-test, $p < 0.05$).
}
\label{tab:quantitative-result}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Results and Discussion}

A summary of the results is presented in Table \ref{tab:quantitative-result}.
Using the model described in Section \ref{ssec:experiment-proposed}, we achieved higher scores than the BERT. 
We evaluated several input settings for combinations of cropped images. 
In line with our analysis of visual conversation types, feeding a cropped image of either the corresponding speaker/respondent/object seemed effective. 
In particular, cropped object images, which achieved an accuracy of 86.04\%, obtained higher scores than BERT. Our analysis showed that this was related to the \textit{visual question-answering} type, where BERT failed to capture the context from the provided text. 
In comparison with LXMERT's results, our model with a cropped image as an additional input feature also resulted in better scores. 
Please note that, as described in Section \ref{ssec:experiment-transformer}, because LXMERT treats an image as a set of region features, we fed the image and focused on the region parts (speaker, respondent, and topic object) as the region features and not as cropped images.
In our experiments, we have two variants of inputs for LXMERT: (1) Original image with focused-on Speaker, Respondent, and Object (SRO) regions, and (2) Original image without focused part, instead, we used all object annotations by Visual Genome which typically contains ten or more region features. Table \ref{tab:quantitative-result} shows the corresponding results for these input settings.
In addition to our experiments, we used the same image features for both LXMERT and our model; we did not use Faster-RCNN to extract the image features for LXMERT.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 6
\begin{figure}[b]
\begin{center}
\includegraphics{29-1ia7f6.pdf}
\end{center}
\textbf{Utterance}: Hey dude, look at that pretty girl watching us.\newline
\textbf{Positive}: They are under age man. \newline
\textbf{Negative}: You are going to bust your ass. 
\hangcaption{Cropped object image (yellow box) provides more information for understanding the conversation context.}
\label{fig:scene-understanding}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Our experiments with two cropped images showed more improvement in terms of accuracy and F1 score, particularly with \textit{speaker} and \textit{object} components.This improvement  conformed to our visual conversation categories of the \textit{image-referring response} and \textit{scene-understanding} types. In Figure \ref{fig:scene-understanding}, we can see one example where our proposed model correctly predicted the pair as correct, while BERT failed to do so.
Using two cropped images, in particular, the \textit{speaker} and \textit{object}, can help the model to focus on the important parts. 
Furthermore, in some cases, the \textit{utterance} and \textit{response} texts might refer to different foci. For example, in both pictures in Figure \ref{fig:visual_conv_type}, the topics of the utterance and response might appear unrelated. Therefore, having two related components could help the model to learn the relationship between each text pair and the visual information better.
This setting involving \textit{speaker} and \textit{object} components can be seen as a natural conversation scene, considering that, in actual conversations, the respondent occasionally needs to see the partner and/or the object of the topic to understand the context.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 7
\begin{figure}[b]
\begin{center}
\includegraphics{29-1ia7f7.pdf}
\end{center}
\textbf{Utterance}: Will our bikes be safe?.\newline
\textbf{Positive}: Did you lock yours? \newline
\textbf{Negative}: Do you have a key to unlock it.
\hangcaption{Example case when both BERT and our model failed to predict the positive response correctly.}
\label{fig:failed-case}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Regarding BERT's performance, according to our findings, 
we postulate that when the responses could be perceived as semantically following the utterances, it would be sufficient for BERT to predict the label. 
As described in Section \ref{ssec:experiment-transformer}, in Figure \ref{fig:error_analysis}, BERT predicted the label correctly, whereas LXMERT, with visual information, failed to do so. However, our model with \textit{speaker} and \textit{object} (a sweater) cropped images can also correctly predict the label. 
We argue that since our model had a clearer object-related visual clue --- a cropped image compared with the bounding boxes of LXMERT --- it could associate the textual context of ``warm'' even though the scene was arguably ``cold.''
Therefore, we argue that to capture a better context for the conversation, visual scene information is essential, particularly for the three visual conversation types: visual question-answering, image-referring response, and scene-understanding. 
Nevertheless, we found out that in some particular cases, for example, in Figure \ref{fig:failed-case}, even with the visual scene information, the model could not correct BERT's errors. 
This failure might have been caused by an inability to associate the visual clue of ``bikes'' with the response text.

As negative pairs were generated from conversations with the same images, we also analyzed the sampled results from false-positive cases. We performed this analysis to investigate whether the models' failure was caused by ``naturally positive'' conversations that made it difficult for them to predict the negative label. We found that some false-positive predictions were produced by negative pairs, which can be considered a ``natural positive conversation,'' especially if the input utterance was written in an open-question manner. However, the models are supposed to consider not only the text but also the visual clues with different focused parts. For example, different topic objects are very rare once visual contexts are considered. Because the number of false-positive answers is significantly smaller than the true-negative ones, for example, 600 vs. 2700 for LXMERT or 400 vs. 2800 for our models, we argue that the negative samples worked as intended and the failures were not mainly caused by false-positive cases.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion and Future Work}
\label{sec:conclusion}

In this study, we investigated how to improve visual conversation tasks using scenic images. 
We presented a new dataset as a means of finding the answer. 
Our findings show that visual information is necessary when context understanding is required.
We also identified three types of visual conversations where visual conversation scenes are essential to understanding the conversation context: (1) the \textit{visual question-answering} type, (2)  \textit{image-referring response} type, and (3) \textit{scene-understanding} type.
Based on this analysis, we also proposed an approach to using scene components in the tasks, particularly the speaker, respondent, and topic object parts. 
Our model was able to significantly outperform the baseline models by partially coping with the first and second conversation types, with 91\% accuracy.
Our results suggest interesting and promising directions for future research.  We predict that focusing on the second and third visual conversation types should be fruitful for many years to come.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.6}
\bibliography{07refs}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Abdurrisyad Fikri}{%
Abdurrisyad Fikri received his B.Sc. in informatics from the Bandung Institute of Technology in 2012. He worked as a web application developer from 2013 to 2016 at Imani Prima, Indonesia. He received M. Eng. from the Tokyo Institute of Technology in 2018. He is currently a Ph.D. student at the Department of Information and Communications Engineering, Tokyo Institute of Technology. His current research interests include dialogue models and multimodal natural language processing.
}

\bioauthor[:]{Hiep V. Le}{%
Hiep V. Le received his B.Eng. in Computer Science from the Ho Chi Minh City University of Technology in 2015, and an M.Eng. in Artificial Intelligence at the Tokyo Institute of Technology in 2018. He is currently working at LINE Fukuoka Corporation. His current research interests include large-scale machine learning in computer vision and natural language processing.
}

\bioauthor[:]{Takashi Miyazaki}{%
Takashi Miyazaki received his Ph.D. degree in computer science from Keio University. He then studied neuroscience as a postdoctoral researcher at the National Institute for Physiological Sciences and the University of Tokyo. Since 2011, he has been working at Yahoo Japan Corporation. His current research interests include computer vision and natural language processing.
}

\bioauthor[:]{Manabu Okumura}{%
Manabu Okumura was born in 1962. He received his B.E., M.E., and Dr. Eng. from the Tokyo Institute of Technology in 1984, 1986, and 1989, respectively. He was an assistant at the Department of Computer Science, Tokyo Institute of Technology, from 1989 to 1992, and an associate professor at the School of Information Science, Japan Advanced Institute of Science and Technology from 1992 to 2000. He is currently a professor at the Institute of Innovative Research, Tokyo Institute of Technology. His current research interests include natural language processing, especially text summarization, computer-assisted language learning, sentiment analysis, and text data mining.
}

\bioauthor[:]{Nobuyuki Shimizu}{%
Nobuyuki Shimizu is a researcher at Yahoo Japan Corporation.  His research topics include natural language, image processing, and artificial intelligence. Ph.D. (computer science) Prior to joining Yahoo Japan Corporation in 2011, he was a project lecturer at the Information Technology Center at the University of Tokyo.
}

\end{biography}

\biodate



\end{document}

