\documentclass[english,tombow]{jnlp_3.3}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{url}
\usepackage{gentombow}
\settombowbleed{5mm}
\usepackage{hangcaption_jnlp}
%%%%\usepackage{udline}
%%%%\setulminsep{1.2ex}{0.2ex}
%%%%\let\underline=\ul
\allowdisplaybreaks


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
%%%%\usepackage{amsmath}
%%%%\usepackage{url}
%%%%\usepackage{multirow}
%%%%\usepackage{color}
%%%%\newcommand{\red}[1]{\textcolor{red}{#1}}
\newcommand{\tabref}[1]{Table \ref{#1}}
\newcommand{\figref}[1]{Figure \ref{#1}}
\newcommand{\figrefabbrev}[1]{Fig. \ref{#1}}
\newcommand{\mysecref}[1]{Section \ref{#1}}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\Volume{28}
\Number{1}
\Month{March}
\Year{2021}
%%%%\Issuetitle{「　　　　」特集号}

\received{2020}{8}{1}
\revised{2020}{11}{8}
%%%%\rerevised{2006}{5}{6}
\accepted{2020}{12}{12}

\setcounter{page}{136}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%   Title, Author, etc.            %%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Collection of Meta Information with User-Generated Question Answer Pairs and its Reflection for Improving Expressibility in Response Generation}
\eauthor{Takashi Kodama\affiref{Author_1} \and Ryuichiro Higashinaka\affiref{Author_2} \and Koh Mitsuda\affiref{Author_3} \and Ryo Masumura\affiref{Author_3} \and Yushi Aono\affiref{Author_3} \and Ryuta Nakamura\affiref{Author_4} \and Noritake Adachi\affiref{Author_4} \and Hidetoshi Kawabata\affiref{Author_4}}
\eabstract{
This paper concerns the problem of realizing consistent personalities in neural conversational modeling by using user generated question-answer pairs as training data. Using the framework of role play-based question-answering, we collected single-turn question-answer pairs for particular characters from online users. Meta information was also collected such as emotion and intimacy related to question-answer pairs. We verified the quality of the collected data and, by subjective evaluation, we also verified their usefulness in training neural conversational models for generating responses reflecting the meta information, especially emotion.
}
\ekeywords{Data Collection, Meta Information, User-generated Content, Utterance \linebreak Generation, Question-Answering}

\headauthor{Kodama et al.}
\headtitle{Collection of Meta Information with User-Generated Question Answer Pairs}

\affilabel{Author_1}{}{Kyoto University, Graduate School of Informatics; this is the work done during the internship at NTT.}
\affilabel{Author_2}{}{NTT Corporation; Nagoya University, Graduate School of Informatics}
\affilabel{Author_3}{}{NTT Corporation}
\affilabel{Author_4}{}{DWANGO Co., Ltd.}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}

Much attention has been paid to non-task oriented dialogue systems from their social and entertainment aspects \cite{Wallace2009,banchs-li-2012-iris,higashinaka-etal-2014-towards}. To make such systems more engaging, it is necessary that they exhibit consistent personalities. In neural conversational modeling \cite{sordoni-etal-2015-neural,vinyals2015neural,shang-etal-2015-neural,serban-etal-2016-building,adiwardana2020humanlike,Roller2020RecipesFB}, various architectures have been proposed to realize consistent personalities, such as incorporating user IDs \cite{li-etal-2016-persona}, personal attributes \cite{ijcai2018-595}, and profile text \cite{zhang-etal-2018-personalizing}. However, the problem is that it is still costly to collect dialogue data related to particular personalities for training such neural models. 

In this paper, we deal with single question-answer pairs as dialogue data. For efficiently collecting such dialogue data for a particular character, a data collection method called ``role play-based question-answering'' has been proposed \cite{higashinaka2013using,higashinaka-etal-2018-role} (See \mysecref{sec:narikiri} for details). In this method, fans of a particular character voluntarily provide question-answer pairs by playing the role of that character. It has been demonstrated that high-quality question-answer pairs can be efficiently collected and that dialogue systems that exhibit consistent personalities can be realized \cite{higashinaka-etal-2018-role}.

This paper extends this work and aims to collect question-answer pairs for particular characters together with other pieces of information (called ``meta information''), such as emotion and intimacy levels. The aim of collecting this additional data is to realize dialogue systems whose utterances can be controlled to reflect such meta information \cite{zhou2018emotional,zhou-wang-2018-mojitalk,song-etal-2019-generating}. This is a useful feature when we want to realize systems that are affective and can become more intimate as an interaction progresses \cite{zhou-wang-2018-mojitalk}. We verify the quality of the collected data and empirically show that conversational models that exhibit consistent personalities as well as meta information, especially emotion, can be successfully realized by using voluntarily provided user-generated question-answer pairs.
Note that, in this paper, we regard personality as the character-ness of a real or fictional character.

In what follows, we first describe the idea of role play-based question-answering followed by our data collection of question-answer pairs and meta information. Then, in Section 3, we describe our approach for training conversational models that take the meta information into account. In Section 4, we describe experiments conducted to verify our approach; we performed both objective and subjective evaluations using the data collected for three characters. Finally, we summarize the paper and mention future work.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Data Collection using Role Play-based Question-answering}
\label{sec:narikiri}

Role play-based question-answering \cite{higashinaka2013using,higashinaka-etal-2018-role} is a data collection framework in which multiple users (typically fans) play the role of a certain character and respond to questions by online users (who can also be fans). Since the fans are knowledgeable about the character and find it amusing to answer questions in the role of their favorite character and online users can ask various questions to their favorite character, this framework can motivate users to voluntarily provide dialogue data centering around a particular character. \citeA{higashinaka-etal-2018-role} showed that fans are highly motivated to provide data and that the collected data are of high quality to realize dialogue systems exhibiting consistent personalities.

In this study, we use this framework to collect question-answer pairs together with other pieces of meta information. More specifically, we collect emotion and intimacy levels from fans in addition to question-answer pairs.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Data collection including meta information}

We collected dialogue data (single-turn question-answer pairs) for three famous characters in Japan: Ayase Aragaki (Ayase), Hime Tanaka (Hime), and Hina Suzuki (Hina). Ayase is a fictional character in the novel series ``Ore no imouto ga konnani kawaii wakeganai'' (My Little Sister Can't Be This Cute). Her character is often referred to as a ``yandere.'' According to Wikipedia, Yandere characters are mentally unstable, incredibly deranged, and use extreme violence or brutality as an outlet for their emotions. Hime and Hina are virtual YouTubers and form a duo called ``HIMEHINA.'' Hime's character is friendly, and Hina has a goofy and laid-back character.

Question-answer pairs were collected on the websites established in the characters' fan communities. Figures \ref{fig:ayase_web_cite} and \ref{fig:hime_web_cite} show screenshots of the websites for Ayase and Hime, respectively. The website for Hina is identical to that for Hime except that the images used were those of Hina. Users can ask the characters questions by using a text-field interface, and users who want to play the role of the characters can post answers. Users can post questions and answers at any time; that is, the interaction is asynchronous. Multiple answers can be posted to the same question. In addition, users can input meta information at the same time when posting their answers. The meta data that we collected were of two kinds:

\begin{description}
\item[Emotion] is a label provided for an answer. This indicates the emotion behind the answer, such as angry or happy. The list of emotions is different for each character. There are 10 types of emotion labels for Ayase, including ``Normal,'' ``Stumped,'' and ``Angry.'' The labels were decided on the basis of emotions that she exhibits in the novel series in which she appears. For Hime and Hina, we had slightly different emotion labels decided on the basis of their behavior on YouTube. We employed the notion of basic emotions for ease of annotation for online users \cite{ekman1992argument}.

\item[Intimacy] is a label provided for an answer. It indicates how close each respondent is feeling to the questioner, and its value is discrete, taking one integer from 1 (least intimate; intimacy level for a stranger) to 5 (most intimate; intimacy level for a family member).
\end{description}

Since the intimacy feature had not been developed when collecting data for Ayase, we collected only emotion labels for Ayase and collected both kinds of meta information for Hime and Hina. \tabref{tab:qa_expamles} shows the collected data examples.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1
\begin{figure}[t]
\begin{center}
\includegraphics{28-1ia6f1.pdf}
\end{center}
\caption{Website for Ayase for collecting question-answer pairs}
\label{fig:ayase_web_cite}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 2
\begin{figure}[t]
\begin{center}
\includegraphics{28-1ia6f2.pdf}
\end{center}
\caption{Website for Hime for collecting question-answer pairs}
\label{fig:hime_web_cite}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Statistics of collected data}

The statistics of collected question-answer pairs are shown in \tabref{tab:Statistics_QA_pairs}. There were 15,179, 12,746, and 10,739 pairs collected for Ayase, Hime, and Hina, respectively. They were collected within a period shorter than one month, indicating that the framework is efficient for collecting dialogue data. Note that the users who provided data were not paid for their effort; the work was totally voluntary.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[t]
\input{06table01.tex}
\hangcaption{Examples of collected data. Numbers in ``Meta'' column indicate intimacy level. The questions and answers are originally in Japanese and were translated by the authors.}
\label{tab:qa_expamles}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 2
\begin{table}[t]
\input{06table02.tex}
\caption{Statistics of collected question-answer pairs for Ayase, Hime, and Hina.}
\label{tab:Statistics_QA_pairs}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

As for the meta information, distributions of emotion labels for Ayase are shown in Figure \ref{fig:distribution_emotion_ayase}. Those for Hime and Hina are shown in Figure \ref{fig:distribution_emotion_hime_and_hina}. It can be seen that the emotion labels for Ayase are rather equally distributed, whereas ``Joyful'' seems dominant for Hime and Hina, representing their personality. The distributions of intimacy labels for Hime and Hina are shown in Figure \ref{fig:distribution_intimacy_hime_and_hina}. For both characters, there were few ``1'' labels. The dominant labels were ``3'' and ``5.''

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 3
\begin{figure}[t]
\begin{center}
\includegraphics{28-1ia6f3.pdf}
\end{center}
\caption{Distribution of emotion labels for Ayase}
\label{fig:distribution_emotion_ayase}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 4
\begin{figure}[t]
\begin{center}
\includegraphics{28-1ia6f4.pdf}
\end{center}
\caption{Distributions of emotion labels for Hime and Hina}
\label{fig:distribution_emotion_hime_and_hina}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 5
\begin{figure}[t]
\begin{center}
\includegraphics{28-1ia6f5.pdf}
\end{center}
\caption{Distributions of intimacy labels Hime and Hina}
\label{fig:distribution_intimacy_hime_and_hina}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Data quality}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Procedure}
\label{user_generated}

To confirm the quality of the collected question-answer pairs with online users, 
we conducted a subjective evaluation by using human judges who do not include the authors. We asked potential judges to answer a questionnaire on a five-point Likert scale (1: not knowledgeable, 5: very knowledgeable) for their knowledge levels about each character, and only used those whose self-declared knowledge level was three or more. As a result, eleven, nine, and nine judges participated in the evaluation of question-answer pairs for Ayase, Hime, and Hina, respectively.

The judges rated each answer by their degree of agreement to the following statements on a five-point Likert scale (1: strongly disagree, 5: strongly agree).

\begin{description}
\item[Naturalness]: The answer is appropriate for the character's response.
\item[Reflection]: The answer reflects the meta information (emotion or intimacy).
\end{description}

When judging the naturalness, the judges were shown pairs of a question and a user-generated answer. 100 unique question-answer pairs were randomly selected from the collected question-answer pairs for this evaluation.

When judging the reflection of meta information, the judges were shown a tuple of a question, the meta information, and the user-generated answer. 100 unique tuples were randomly selected from the collected data for this evaluation. As a control, we prepared 100 unique tuples with meta information randomly replaced
with different meta information.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Results}
\label{sec:quality_results}

\tabref{tab:ugc_eval} shows the evaluation results. 
\pagebreak
In terms of naturalness, all three characters attained high scores. This shows that even though role play-based question-answering does not pay users for their efforts, it can be used to collect appropriate responses for characters. This conforms to the results shown in \cite{higashinaka-etal-2018-role}.
Here, we define characteristic responses as those having a naturalness score of 3 or more. Under this definition, the percentages of characteristic responses in the collected data for Ayase, Hime, and Hina were 86\%, 99\%, and 98\%, respectively. This indicates that the responses are of high naturalness.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[b]
\input{06table03.tex}
\hangcaption{Results of human evaluation for collected question-answer pairs. Scores were averaged over all judges. Standard deviations are shown in parentheses.}
\label{tab:ugc_eval}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 6
\begin{figure}[b]
\begin{center}
\includegraphics{28-1ia6f6.pdf}
\end{center}
\caption{Score differences between judges}
\label{fig:score_diff}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

For the reflection of emotion, when we look at ``Actual,'' we seem to have good quality emotion labels. When we compare ``Actual'' vs. ``Random'' (randomly replaced emotion labels), we have a good amount of drop, meaning that the utterances and emotions are well associated in our data. As for the intimacy, the results were different. Although the ``Actual'' scores were high, the results for ``Random'' also exhibited high scores (although with a slight drop), meaning that the utterances and intimacy levels are not as associated when compared with the case of emotion; it may be difficult for humans to accurately recognize the level of intimacy from utterances.

Figure~\ref{fig:score_diff} shows the proportion of score differences between judges (i.e., score differences between all judge pairs) for naturalness and reflection of meta information. As for meta information, we only used the results for ``Actual'' for this analysis. Overall, we can see that most score differences are below 1 or less, meaning that the decisions of the judges are similar to each other.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Training Conversational Models that Reflect Meta Information}

To test whether it is possible to generate utterances that reflect meta information collected by role-play based question answering, using the collected question-answer pairs as well as meta information, we train neural conversational models. Figure \ref{fig:overview} illustrates the overview of our training procedure. Since we want to generate utterances that reflect meta information, we adopt a model architecture that can take such additional information into account in decoding. Below, we explain how we train such conversational models.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 7
\begin{figure}[b]
\begin{center}
\includegraphics{28-1ia6f7.pdf}
\end{center}
\caption{Overview of our approach}
\label{fig:overview}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Pre-training with meta information}

The amount of data collected for Ayase, Hime, and Hina may be too small to learn a generation model from scratch. Therefore, we decided to pre-train the models with a large amount of data. In this study, we used the large number of general question-answer pairs that we collected previously when developing our question answering system \cite{question-answers-higashinaka2013}. This dataset, ``General QA pairs,'' was collected via crowdsourcing. Crowdworkers were given topics and wrote questions and answers related to the topics; each human intelligence task (HIT) asked each worker to provide 10 question-answer pairs for a topic. There are 500K question-answer pairs in this dataset. For example, for the topic Mt. Fuji, the dataset includes question-answer pairs such as Q: ``Do you know the height of Mt. Fuji?'' and A: ``It's 3,776 meters,'' Q: ``Is Mt. Fuji the highest mountain in Japan?'' and A: ``Yes, it is.''

Meta information (i.e., emotion and intimacy labels) is not included in General QA pairs, which may be problematic in the later fine-tuning stage because pre-training without meta information might lead to models that ignore the meta information. Therefore, we trained classifiers for meta information from our data for Ayase, Hime, and Hina and automatically annotated General QA pairs with such meta information. We call the dataset annotated in this way ``Auto-annotated general QA pairs.'' By performing pre-training using Auto-annotated general QA pairs, a model is likely to take into account the meta information appropriately when fine-tuning with the data with meta information.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Generation models}

Currently, pre-trained language models are showing promising results in a wide variety of natural language processing tasks \cite{devlin-etal-2019-bert,radford2019language,yang2019xlnet,Liu2020MultilingualDP}. Such models can more accurately capture the meaning of words depending on the context with a massive amount of training data, enabling them to be applied to fine-tuning for particular downstream tasks. Since BERT (Bidirectional Encoder Representations from Transformers), which is a pre-trained language model, has been found to be useful for generation tasks \cite{zhang-etal-2019-pretraining}, we use it in our work. Specifically, we adopt the dual-source BERT encoder-decoder model \cite{junczys-dowmunt-grundkiewicz-2018-ms,correia-martins-2019-simple}. The model allows for the incorporation of additional information. The model was originally proposed for automatic post-editing in machine translation. It takes two inputs: source text in the source language and a tentative machine translation result for that text as additional information. It then outputs target text in the target language. We use this model for our generation models. In this study, as additional information, we used meta information instead of tentative machine translation results.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Classifiers for meta information}
\label{sec:classifiers}

We need classifiers for meta information for creating Auto-annotated general QA pairs. To realize the classifiers, we created BERT-based classifiers with an additional multi-layer perceptron (MLP) layer, using the representation encoded by BERT as input. We used
bert-base-multilingual-cased\footnote{\url{https://github.com/google-research/bert}}
as the BERT model. Tokenization was done by using a SentencePiece\footnote{\url{https://github.com/google/sentencepiece}} \cite{kudo-richardson-2018-sentencepiece} model trained with Japanese Wikipedia. The vocabulary size was 32K.

For emotion, there were 10 classes for Ayase and 8 classes for Hime and Hina. In the case of intimacy level, there were five classes for Hime and Hina. For all classifications, we used the cross-entropy as the loss function. We trained the models with a batch size of 32 and a learning rate of 0.00002 for 3 epochs.\footnote{We followed the settings as shown in \url{https://github.com/huggingface/transformers}.} The classifiers were trained by using the collected data for Ayase, Hime, and Hina and were applied to General QA pairs to create Auto-annotated general QA pairs.

For Ayase, when training the classifiers, we first split the Ayase data randomly into 9/1 for training/test\footnote{Note that the training data here correspond to the training and development data in Section \ref{sec:experiments}.} and trained a classifier by using the training set. The accuracy of our classifier was 43.4\% using the test set. Compared with the majority baseline that labels all examples as ``Normal'' and whose accuracy was 30.4\%, we considered this accuracy to be reasonable. As for Hime and Hina, we also split the data randomly into 9/1 for training/test. The training procedure was the same as that for Ayase. The classifiers were created for each kind of meta information, that is, emotion and intimacy. The accuracies of the emotion classifiers (majority baseline accuracies in parentheses) for Hime and Hina were 62.1\% (54.6\%) and 63.7\% (59.1\%), respectively. The accuracies of the intimacy classifiers for Hime and Hina were 40.5\% (33.4\%) and 41.2\% (36.7\%), respectively. Although the accuracies are not that high, we consider them to be reasonable since they outperformed the baselines. The classification accuracies for emotion were much higher than those for intimacy, probably because it is more difficult to distinguish intimacy levels as discussed in Section \ref{sec:quality_results}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 4
\begin{table}[b]
\input{06table04.tex}
\hangcaption{Distribution of automatically annotated labels for Ayase's emotion. The numbers denote the number of utterances.}
\label{tab:auto_ayase_emo}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


Tables \ref{tab:auto_ayase_emo}, \ref{tab:auto_hime_hina_emo}, and \ref{tab:auto_hime_hina_inti} show the distributions in automatic annotation for Ayase's emotion, Hime's and Hina's emotion, and Hime's and Hina's intimacy. Although general QA pairs are mainly annotated with ``Normal'' for emotion and ``3'' or ``5'' for intimacy, there are quite a few kinds of labels appearing in the classifier results; e.g., the proportion of Hina's ``Joyful'' was 35.7\% (= 176,150/493,348), making the classification results possibly useful as training data for reflecting emotion/intimacy.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 5
\begin{table}[t]
\input{06table05.tex}
\hangcaption{Distribution of automatically annotated labels for Hime's and Hina's emotion. The numbers denote the number of utterances.}
\label{tab:auto_hime_hina_emo}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 6
\begin{table}[t]
\input{06table06.tex}
\hangcaption{Distribution of automatically annotated labels for Hime's and Hina's intimacy. The numbers denote the number of utterances.}
\label{tab:auto_hime_hina_inti}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experiments}
\label{sec:experiments}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Models for comparison}

We trained our conversational models and evaluated their performance. We trained three models for comparison:

\begin{description}
\item[wo-Meta\textmd{:}]pre-trained using General QA pairs (without meta information) and fine-tuned without meta information using role play-based QA pairs.
\item[w-Meta\textmd{:}]pre-trained using General QA pairs (without meta information) and fine-tuned with meta information using role play-based QA pairs.
\item[w-Meta+Anno\textmd{:}]pre-trained using Auto-annotated general QA pairs (with meta information) and fine-tuned with meta information using role play-based QA pairs.
\end{description}

We assumed that, by comparing the results for wo-Meta and w-Meta, we could check whether the meta information collected during role play-based question-answering was useful in generating responses that reflect the meta information. By comparing w-Meta and w-Meta-Anno, we could check whether the automatic annotation of meta information was useful for pre-training models. Note that the aim of this paper is to verify whether utterances that reflect meta information can be generated with user-generated question-answer pairs. We used OpenNMT-APE\footnote{\url{https://github.com/deep-spin/OpenNMT-APE}} for training the models with default parameters. OpenNMT-APE implements the dual-source BERT encoder-decoder model that allows for the incorporation of additional information. We used the same BERT and
sentencepiece models as in Section~\ref{sec:classifiers}. The collected data were randomly split into
train, development, and test sets with the ratio of 8/1/1 in this experiment.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Automatic evaluation}

Tables \ref{tab:auto_eval_emo} and \ref{tab:auto_eval_inti} show the results of the automatic evaluation against the test data for emotion and intimacy, respectively. We used perplexity, distinct-1,2, and BLEU-1,2,3,4 as evaluation metrics \cite{liu-etal-2016-evaluate}. Perplexity measures the adequacy of language models. Distinct metrics measure the diversity of expressions in generated utterances, and BLEU metrics measure the accuracy of generated utterances in terms of lexical overlaps with references.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 7
\vspace{0.5\Cvs}
\begin{table}[b]
\input{06table07.tex}
\caption{Results of automatic evaluation for emotion}
\label{tab:auto_eval_emo}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 8
\begin{table}[b]
\input{06table08.tex}
\caption{Results of automatic evaluation for intimacy}
\label{tab:auto_eval_inti}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

From the tables, we can see that, for emotion, w-Meta+Anno seems to have achieved good results for Ayase and Hina, although the results were mixed for Hime. Across the characters, it can be seen that w-Meta+Anno performed the best for Distinct-2, indicating that our pre-training with Auto-annotated general QA pairs had positive effects in terms of improving the variety in the utterances. For intimacy, the tendencies for Hime and Hina were different with no particular models outperforming others 
except w-Meta+Anno for Distinct-1 and Distinct-2. This is in line with the results for emotion.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Human evaluation}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Procedure}

To assess the quality of the generated responses, we conducted a subjective evaluation. The procedure for this evaluation was the same as that in Section~\ref{user_generated}. We used judges whose self-declared knowledge level about the characters was three or more. As a result, eleven, twelve, and twelve judges participated in the evaluation of question-answer pairs for Ayase, Hime, and Hina, respectively. Note that we used a different set of judges from the experiment in Section~\ref{user_generated}. The judges rated each output answer by their degree of agreement to the following statements on a five-point Likert scale (1: strongly disagree, 5: strongly agree).

\begin{description}
\item[Naturalness\textmd{:}] The answer is appropriate for the character's response.
\item[Reflection\textmd{:}] The answer reflects the meta information (emotion or intimacy).
\end{description}

When judging the naturalness, the judges were shown pairs of an input question and an answer output by each of the generation models. The input to the models was the 100 questions (with meta information when the model requires it) randomly sampled from the test data. The answers for a question to be evaluated were randomly shuffled and presented. We asked the judges to evaluate each output independently and give the same score if the generated response to a question was the same.

When judging the reflection of meta information, the judges were shown a tuple of an input question, the meta information, and the output answer. We prepared two sets of 100 questions as input to the models. One set comprised random samples from the test data. The other set also comprised random samples from the test data, but unseen meta information was used; this set was created by artificially replacing the meta information with different meta information. For instance, when a tuple had an input question (``Do you like Hime!?'') and the meta information (``Favorable''), the meta information was forcibly replaced with another piece of meta information (e.g., ``Surprised''), which was randomly selected from labels excluding the original label (``Favorable''). By using this set, it was possible to test the robustness of the models for unseen (possibly discrepant) meta information. We call the former condition ``Seen'' and the latter ``Unseen.''


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Results for naturalness}

Table \ref{tab:human_eval_natural} shows the results of the human evaluation for naturalness. Among the three models we trained, w-Meta had the best score. However, the differences of the models were small and not statistically significant. Here we used a Wilcoxon rank-sum test as a statistical test with Bonferroni correction. The three models seemed to show the same level of naturalness, which is good for w-Meta+Anno because this means it can achieve higher distinct scores without a loss of naturalness.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 9
\begin{table}[b]
\input{06table09.tex}
\caption{Results of human evaluation for naturalness. Scores were averaged over all judges.}
\label{tab:human_eval_natural}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The naturalness of generated responses needs to be sufficiently high in order to guarantee the naturalness of utterances that incorporate meta information such as emotion and intimacy.
\tabref{tab:ugc_eval} shows that, in the collected data, the naturalness of the responses for Ayase, Hime, and Hina were 3.62, 3.97, and 3.99, respectively. Since the data evaluated here were those made by humans, the figures are considered to be the upper bound of the naturalness. The naturalness of the responses generated by the proposed models (w-Meta and w-Meta+Anno) were 3.33, 3.76, and 3.68, respectively, at the lowest, as shown in \tabref{tab:human_eval_natural}. These scores reached 90\% or more in relation to the upper bound. This high percentage indicates that the naturalness of the generated responses was sufficiently high.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 10
\begin{table}[b]
\input{06table10.tex}
\hangcaption{%
Results of human evaluation for reflection of emotion. Scores were averaged over all judges. Asterisks (**) indicate whether the value is significantly better than that of ``wo-Meta'' (p $<$ 0.01). Wilcoxon rank-sum test was used as statistical test with Bonferroni correction.}
\label{tab:human_eval_emo_split}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Results for reflection of emotion}

\tabref{tab:human_eval_emo_split} shows the results of the human evaluation for the reflection of emotion. 
We can see that w-Meta+Anno performed the best. For Ayase and Hime, w-Meta+Anno significantly outperformed wo-Meta. For Hina, we did not see a significant difference but the performance was better than wo-Meta. This indicates the effectiveness of our pre-training with Auto-annotated general QA pairs, at least for emotion. When comparing the results of ``Seen'' and ``Unseen,'' the scores for ``Seen'' were generally higher than those for ``Unseen,'' which is reasonable, indicating the difficulty of handling a combination of utterances with unseen emotions.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 11
\begin{table}[b]
\input{06table11.tex}
\hangcaption{%
Results of human evaluation for reflection of each emotion label for Ayase. Scores were averaged over all judges. The numbers in square brackets indicate the number of samples for each emotion label in the test data. Asterisks (*)/Daggers ($^{\dagger}$) indicate whether the value is significantly better/worse than ``wo-Meta.'' */$^{\dagger}$ indicates statistical significance p $<$ 0.05, and ** indicates p $<$ 0.01. Wilcoxon rank-sum test was used as statistical test with Bonferroni correction.}
\label{tab:human_eval_ayase_emo}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 12
\begin{table}[b]
\input{06table12.tex}
\hangcaption{Results of human evaluation for reflection of each emotion for Hime. Scores were averaged over all judges. See Table~\ref{tab:human_eval_ayase_emo} for the notation of the table.}
\label{tab:human_eval_hime_emo}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 13
\begin{table}[b]
\input{06table13.tex}
\hangcaption{Results of human evaluation for reflection of each emotion for Hina. Scores were averaged over all judges. See Table~\ref{tab:human_eval_ayase_emo} for the notation of the table.}
\label{tab:human_eval_hina_emo}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Tables \ref{tab:human_eval_ayase_emo}, \ref{tab:human_eval_hime_emo} and \ref{tab:human_eval_hina_emo} show human evaluation scores for each emotion label for Ayase, Hime and Hina, respectively. Here, we used their ``Seen'' data for analysis.
Note that the numbers of samples for each emotion label are different representing the distribution of the original data. As for Ayase, w-Meta+Anno shows a significant improvement in ``Angry'' and ``Happy'' over wo-Meta. Concerning Hime, w-Meta+Anno significantly outperformed wo-Meta in ``Surprised'' and ``Angry.'' These two characters seem to have achieved their overall improvement by successfully exhibiting several core emotions.

In contrast, there was no particular emotion gaining improvement for Hina. The limited performance of Hina is presumably because Hina had few characteristic phrases to learn, whereas Ayase and Hime had some typical phrases related to specific emotions (e.g, ``I will call the police!'' (``Angry'' for Ayase) and ``W-What is that!?'' (``Surprised'' for Hime)). \figref{fig:emotion_unique_token} shows the percentages of unique tokens found in the responses for each emotion label. With regards to ``Angry'' of Ayase and ``Surprised'' for Hime, the percentage of ``Generated'' is much lower than that of ``Gold''. This suggests that the models learned typical phrases for these emotions. In contrast, Hina tends to have smaller drops on all emotion labels, suggesting that the model have not obtained the typical expression of each emotion.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 8
\begin{figure}[t]
\begin{center}
\includegraphics{28-1ia6f8.pdf}
\end{center}
\hangcaption{Percentages of unique tokens for each emotion. ``Gold'' indicates the gold responses for test data and ``Generated'' indicates the generated responses for test data.}
\label{fig:emotion_unique_token}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Results for reflection of intimacy}
\label{sec:intimacy_reflection}

\tabref{tab:human_eval_inti_split} shows the results of the human evaluation for the reflection of intimacy. 
We did not observe w-Meta and w-Meta+Anno to be superior. It can be observed that w-Meta is almost the same as wo-Meta and w-Meta+Anno is slightly worse than wo-Meta. Tables~\ref{tab:human_eval_hime_inti} and \ref{tab:human_eval_hina_inti} show human evaluation scores for each intimacy label for Hime and Hina, respectively. Here, we used their ``Seen'' data for analysis. The differences of the models were small and not statistically significant for any intimacy label for both characters.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 14
\begin{table}[b]
\vspace{1\Cvs}
\input{06table14.tex}
\hangcaption{Results of human evaluation for reflection of intimacy. Scores were averaged over all judges. Daggers ($^{\dagger}$) indicate whether the value is significantly worse than that of ``wo-Meta'' (p $<$ 0.05). Wilcoxon rank-sum test was used as statistical test with Bonferroni correction.}
\label{tab:human_eval_inti_split}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 15
\begin{table}[b]
\input{06table15.tex}
\hangcaption{Results of human evaluation for reflection of each intimacy label for Hime. Scores were averaged over all judges. The numbers in square brackets indicate the number of samples for each intimacy label in the test data. No significant differences were found between the models.}
\label{tab:human_eval_hime_inti}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 16
\begin{table}[b]
\input{06table16.tex}
\hangcaption{Results of human evaluation for reflection of each intimacy label for Hina. Scores were averaged over all judges. The numbers in square brackets indicate the number of samples for each intimacy label in the test data. No significant differences were found between the models.}
\label{tab:human_eval_hina_inti}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\figref{fig:intimacy_unique_token} shows the percentages of unique tokens for each intimacy label. Both characters showed a tendency to have small drops in all intimacy labels. It is assumed that their generation models could not learn typical expressions associated with each intimacy label. Another reason may be that it is difficult for humans to correctly recognize the level of intimacy from utterances alone, as discussed in Section~\ref{sec:quality_results}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 9
\begin{figure}[t]
\begin{center}
\includegraphics{28-1ia6f9.pdf}
\end{center}
\caption{Percentages of unique tokens for each intimacy}
\label{fig:intimacy_unique_token}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Summary and Future Work}

The purpose of this study was to verify whether a natural utterance can be realized for a character reflecting meta information (emotion and intimacy) from question-answer pairs and meta information obtained by role play-based question-answering. We performed experiments using the data of three famous characters in Japan. When training the generation models, we proposed the use of pre-training with data automatically annotated with meta information, which generally led to better performance.

As for emotion, subjective evaluation results indicate that utterances reflecting meta information can be generated although some differences were found depending on the characters; it seems difficult for a character who lacks typical expressions to generate appropritate responses reflecting emotion. We also found that it was difficult to generate responses that reflect intimacy. Although we require future investigation, we attribute this to the lack of typical phrases related to intimacy and the way humans recognize intimacy in utterances.

For future work, we want to improve the quality of our generation models since there seems to be much room for improvement when compared to human performance. It may also be interesting to apply other pre-training methods \cite{yang2019xlnet,Liu2020MultilingualDP} as well as to incorporate knowledge of the characters in question \cite{ghazvininejad2018knowledge} in order to enhance the character-ness of the generated utterances. We also want to examine the relationship between the naturalness of a generated response and the degree to which the meta information can be reflected. Although we work on the character-ness of a real or fictional character, we also want to apply our method to personality in general in the future.
It is of our particular interests to investigate whether other kinds of meta-information can be collected and used for response generation. Realizing workable dialogue agents based on our work is one of our next steps, which would require the consideration of emotion/intimacy over multiple turns.

Finally, it should be noted that our proposed method has the risk of being used without the approval of copyright holders. For the appropriate development and application of the proposed method, appropriate procedures for using the characters need to be taken in order to avoid legal and ethical problems.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment

We thank the developers of DWANGO Co.,
Ltd. for creating the role play-based question-answering Web sites.
We also thank the users who participated in role play-based question-answering.
This paper is an extended version of our ealier report \cite{kodama-etal-2020-generating}.
We would like to thank the Journal of Natural Language Processing for English writing check for this article.
We also thank the anonymous reviewers for their insightful comments.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.6}
\bibliography{06refs}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}
\bioauthor[:]{Takashi Kodama}{%
Takashi Kodama received his M.S. degree from the Graduate School of Informatics, Kyoto University in 2020. He is currently a Ph.D. student at the Graduate School of Informatics, Kyoto University. His research interests are focused on natural language processing and dialogue systems.
He is a member of the Association for Natural Language Processing (ANLP).
}
\bioauthor[:]{Ryuichiro Higashinaka}{%
Ryuichiro Higashinaka received his B.A. degree in Environmental Information, Masters in Media and Governance, and Ph.D. from Keio University, Kanagawa, in 1999, 2001, and 2008, respectively. He is currently a professor at Nagoya University. He is a visiting distinguished senior researcher at NTT Media Intelligence Laboratories. His research interests include building question-answering systems and spoken-dialogue systems. From November 2004 to March 2006, he was a visiting researcher at the University of Sheffield, UK. He received the ``Commendation for Science and Technology by the Minister of Education, Culture, Sports, Science and Technology'' in 2016. He is a member of the ANLP, Japanese Society for Artificial Intelligence (JSAI), the Information Processing Society of Japan (IPSJ) and the Institute of Electronics, Information and Communication Engineers (IEICE).
}
\bioauthor[:]{Koh Mitsuda}{%
Koh Mitsuda received his M.S. degree in Engineering from Tokyo Institute of Technology and joined the Nippon Telegraph and Telephone (NTT) Corporation in 2015. He is currently a researcher at NTT Media Intelligence Laboratories and a Ph.D. student at Tsukuba University. He is presently engaged in research on natural language processing and dialogue systems. He is a member of the ANLP and JSAI.
}
\bioauthor[:]{Ryo Masumura}{%
Ryo Masumura received his B.E., M.E., and Ph.D. degrees in engineering from Tohoku University, Sendai, Japan, in 2009, 2011, and 2016, respectively. Since joining NTT Corporation in 2011, he has been engaged in research on speech recognition, spoken language processing, and natural language processing. He received the Student Award and the Awaya Kiyoshi Science Promotion Award from the Acoustic Society of Japan (ASJ) in 2011 and 2013, respectively; the Sendai Section Student Awards and the Best Paper Prize from the Institute of Electrical and Electronics Engineers (IEEE) in 2011; the Yamashita SIG Research Award and the SIG-NL Excellent Paper Award from the IPSJ in 2014 and 2018, respectively; the Young Researcher Award and the Paper Award from the ANLP in 2015 and 2020, respectively; and the Information and Systems Society (ISS) Young Researcher's Award in Speech Field and the ISS Excellent Paper Award from the IEICE in 2015 and 2018, respectively. He is a member of the ASJ, IPSJ, ANLP, IEEE, and the International Speech Communication Association (ISCA).
}
\bioauthor[:]{Yushi Aono}{%
Yushi Aono received his Ph.D. degree in engineering science from Osaka University and joined the NTT Corporation in 1999. After working for NTT Cyber Space Laboratories, NTT Advanced Technology, and the NTT Research and Planning Division, he began working as a project manager (principal researcher) in the Cognitive Information Processing Laboratory at NTT Media Intelligence Laboratories. He is engaged in research on speech recognition and speech synthesis and is a member of the IEICE.
}
\bioauthor[:]{Ryuta Nakamura}{%
Ryuta Nakamura joined the Sony Global Manufacturing \& Operations Corporation in 2008. As a software engineer, he developed game consoles and life electronics devices. He joined Gatebox Inc. in 2016 as a software engineer. He was in charge of marketing management in 2017. He joined Dwango Co., Ltd. in 2018 as a new business in charge and joined the ``Narikiri AI'' project. He currently works as a new business manager since 2020. He has been a member of the Kadokawa Digital Strategy Bureau since 2020.
}
\bioauthor[:]{Noritake Adachi}{%
Noritake Adachi joined Nippon Timeshare Co., Ltd. in 2000. He subsequently joined Dwango Co., Ltd. He was a section manager at a smartphone music site. He is currently engaged as a project manager in a conversation AI creation project called the ``Narikiri AI'' project (2016--present), he was involved in Artificial Intelligence Fundraising (2018), and he is a winner of the Horse Racing Programming Competition ``Denno Prize'' (2015--2018).
}
\bioauthor[:]{Hidetoshi Kawabata}{%
Hidetoshi Kawabata joined Rakuten, Inc. in 2007. He was a Manager for the West Japan Area for Rakuten Market Business till 2011. He joined Dwango Co., Ltd. in 2011. He was in charge of e-commerce business development as a NicoNico Channel sales manager in 2012. He is currently a business manager for the NicoNico Channel and joined the ``Narikiri AI'' project in 2016. He has also been part of the Kadokawa Digital Strategy Bureau since 2019.
}
\end{biography}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\biodate



\end{document}

