\documentclass[english]{jnlp_3.6}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{hangcaption_jnlp}
\usepackage{array}
%%%%\usepackage{udline}
%%%%\setulminsep{1.2ex}{0.2ex}
%%%%\let\underline=\ul
\allowdisplaybreaks


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
\usepackage{booktabs}
\usepackage{multirow}
\usepackage{xspace}
%%%%\usepackage{comment}
%%%%\usepackage{amssymb}
\newcommand{\bleu}{\textsc{bleu}\xspace}
\newcommand{\ubleu}{$\upsilon$\textsc{bleu}\xspace}
\newcommand{\dbleu}{$\Delta$\textsc{bleu}\xspace}
\newcommand{\ruber}{\textsc{ruber}\xspace}
\newcommand{\bert}{\textsc{bert}\xspace}
\newcommand{\bertscore}{\textsc{berts}{\footnotesize core}\xspace}
\newcommand{\ubertscore}{$\upsilon$\bertscore}
\newcommand{\nnrater}{\textsc{nn}-rater\xspace}
%%%%\usepackage{xcolor}
%%%%\newcommand{\yn}[1]{\textcolor{black}{#1}}
    \newcommand{\yn}[1]{#1}
%%%%\newcommand{\Add}[1]{\textcolor{red}{#1}}
%%%%\usepackage{soulutf8,cancel}
%%%%\setstcolor{red}
%%%%\newcommand{\Erase}[1]{\st{#1}}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{30}
\Number{2}
\Month{June}
\Year{2023}
%%%%\Issuetitle{「　　　　」特集号}

\received{2022}{10}{1}
\revised{2023}{1}{13}
\accepted{2023}{2}{20}

\setcounter{page}{531}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Uncertainty-aware Automatic Evaluation Method\\ 
for Open-domain Dialogue Systems}
\eauthor{Yuma Tsuta\affiref{Authors_1} \and Naoki Yoshinaga\affiref{Authors_2} \and Masashi Toyoda\affiref{Authors_2}} 
\eabstract{%
Because open-domain dialogues allow diverse responses, common reference-based metrics for text generation, such as \textsc{bleu}, do not correlate well with human judgments unless we prepare an extensive reference set of high-quality responses for input utterances. In this study, we propose a fully automatic, uncertainty-aware evaluation method for open-domain dialogue systems, $\upsilon$\textsc{bleu}. Our method first collects diverse reference responses from massive dialogue data, annotates their quality judgments by using a neural network trained on automatically collected training data, and then computes weighted \textsc{bleu} using the automatically-retrieved and -rated reference responses. We also employ this method with an embedding-based metric, \textsc{berts}{\footnotesize core}, instead of the word-overlap-based metric, \textsc{bleu}, to absorb surface variations of the reference responses. The experimental results on the meta-evaluation of our evaluation method for dialogue systems based on massive Twitter data confirmed that our method substantially improves correlations between \textsc{bleu} 
    (or \textsc{berts}{\footnotesize core}) and human judgments. 
%%%%(or \textsc{berts}$_{\text{core}}$) and human judgments. 
We also confirmed that our method is effective when it is combined with a reference-free metric.
}
\ekeywords{Open-domain Dialogue System, Evaluation Method}

\headauthor{Tsuta et al.}
\headtitle{Uncertainty-aware Evaluation Method for Open-domain Dialogue Systems}

\affilabel{Authors_1}{}{Graduate School of Information Science and Technology, The University of Tokyo}
\affilabel{Authors_2}{}{Institute of Industrial Science, The University of Tokyo}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

\renewcommand{\thefootnote}{}
\footnote[0]{Part of this study was presented at proceedings of the 58th Annual Meeting of the Association for Computational Linguistics: Student Research Workshop.}
\renewcommand{\thefootnote}{\arabic{footnote}}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}

Interest in intelligent dialogue agents (i.e., Apple Siri, Amazon Alexa, and Google Assistant) has increased. 
The key to achieving higher user engagement with the dialogue agents is to support open-domain non-task-oriented dialogues to return a meaningful response for any user input.
Along with an increase of online conversational data on social media platforms such as Twitter and Reddit and the advent of deep neural networks, researchers study data-driven approaches to open-domain dialogue systems \cite{ritter-etal-2010-unsupervised,DBLP:journals/corr/VinyalsL15,shang-etal-2015-neural}.

The major challenge in developing open-domain dialogue systems is that existing evaluation metrics for text generation tasks, such as \bleu\ \cite{papineni-etal-2002-bleu}, correlate poorly with human judgment in evaluating responses generated by dialogue systems \cite{liu-etal-2016-evaluate}.
In open-domain dialogues, even though responses with various contents and styles are acceptable \cite{sato-etal-2017-modeling}, only a few responses, or often only one, are available as reference responses in evaluation datasets because the datasets are made from actual online conversations.
Therefore, it is hard for these reference-based metrics to consider uncertain responses without writing additional reference responses by hand \cite{li-etal-2017-dailydialog} (\S~\ref{sec:rel}). 

To address the weak correlation between existing reference-based metrics and human evaluation, some researchers resort to reference-free metrics \cite{Tao_Mou_Zhao_Yan_2018,mehri-eskenazi-2020-unsupervised,mehri-eskenazi-2020-usr}, while other researchers augment reference-based metrics with embedding-based soft matching (instead of word overlaps used in \bleu) \cite{Zhang_2020BERTScore} or more directly with multiple automatically-extended reference responses \cite{sordoni-etal-2015-neural}. 
However, the task of evaluating responses without references is inherently as difficult as generating responses themselves and requires extra supervision to obtain a high correlation with human judgments \cite{lowe-etal-2017-towards,sai-etal-2020-improving}.
Although embedding-based soft matching can partially address the diversity of outputs in open-domain dialogues, it is useless to evaluate responses that have different semantic content with available references.
\citeA{galley-etal-2015-deltableu} proposed \dbleu (\S~\ref{sec:delta-bleu}), which integrates human judgments on automatically-extended reference responses with diverse quality in \bleu computation. Although this method shows promising results, it requires human judgments to score automatically-extended references.
Therefore, it cannot effectively evaluate open-domain dialogue systems in a wide range of domains of interest.

In this study, \yn{to obtain robust reference-based metrics for open-dialogue systems, we propose an automatic, uncertainty-aware evaluation metric, \ubleu, which removes the human intervention in \dbleu.} 
The proposed metric exploits reference responses that are retrieved from massive dialogue logs and are automatically rated by a neural network (\S~\ref{sec:method}).
We first retrieve diverse response candidates according to the similarity of utterances to which the responses were directed.
Next, we evaluate the appropriateness of each retrieved response using a neural network.
We train the neural network model using data automatically generated from an utterance with multiple responses.
 Finally, we use the retrieved responses and their automatic scores to evaluate the target response with a weighted \bleu following \citeA{galley-etal-2015-deltableu}.
In addition, we propose two extensions:
first, an improvement of the overall method using \bert\ \cite{devlin-etal-2019-bert} and its derivative, the embedding-based metric \bertscore\ \cite{Zhang_2020BERTScore} (\S~\ref{sec:bertscore});
second, an integration with the evaluation metric \ruber\ \cite{Tao_Mou_Zhao_Yan_2018} (\S~\ref{sec:ruber}), which combines reference-free and reference-based metrics, to demonstrate the usefulness of our method when combined with reference-free metrics.

 
Using our method, we experimentally evaluated responses generated by dialogue systems such as a retrieval-based method \cite{liu-etal-2016-evaluate}, a generation-based method \cite{Serban_Sordoni_Lowe_Charlin_Pineau_Courville_Bengio_2017}, and human as an ideal system\footnote{To use human responses, we collect utterances with multiple responses as a test dataset.} using Twitter dialogues (\S~\ref{sec:exp}). 
Our method is comparable to \dbleu in terms of its correlation with human judgment, and when it is integrated into \ruber\ \cite{Tao_Mou_Zhao_Yan_2018}, it substantially improves the correlation (\S~\ref{sec:res}).

Our contributions are as follows. 
\begin{itemize}
    \item We developed a fully-automatic uncertainty-aware evaluation method for open-domain dialogue systems. Our method automates the human ratings required in \dbleu while maintaining the performance.
    \item We demonstrated that even a recent embedding-based reference-based metric, \bertscore, can benefit from the extended reference responses that are retrieved and then rated by our method.
    \item We confirmed that combining \ubleu with the unreferenced scorer of \ruber\ \cite{Tao_Mou_Zhao_Yan_2018} improves \ruber's performance by considering the diversity of outputs in open-domain dialogues.
\end{itemize}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Work} \label{sec:rel}

Here, we introduce recent studies on evaluating open-domain dialogue systems. 
We focus on model-agnostic methods that can evaluate a system response for a given utterance.\footnote{Perplexity is sometimes used to evaluate dialogue systems \cite{hashimoto-etal-2019-unifying}.
However, it is only applicable to generation-based dialogue systems; therefore, we do not discuss it here, like \citeA{liu-etal-2016-evaluate}.}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Reference-based metrics}

For the evaluation of dialogue systems, researchers have first adopted evaluation metrics that have been designed for other text-generation tasks.
Specifically, \bleu\ \cite{papineni-etal-2002-bleu} and \textsc{meteor} \cite{banerjee-lavie-2005-meteor} for machine translation and \textsc{rouge} \cite{lin-2004-rouge} for automatic summarization are used to evaluate open-domain dialogue systems.
However, these word-overlap-based metrics are hard to capture semantic similarities between words and have been reported to correlate poorly with human judgments \cite{liu-etal-2016-evaluate}.

Embedding-based metrics such as \bertscore\ \cite{Zhang_2020BERTScore} remedy the shortcomings of word-overlap-based metrics by exploiting word embeddings that capture word meanings with continuous vector representations.
Unfortunately, even using embedding-based similarities, reference-based metrics correlate poorly with human judgments in evaluating dialogue systems \cite{liu-etal-2016-evaluate,ji-etal-2022-achieving}.
This is because reference-based metrics are computed with only single reference responses taken from actual conversations, whereas acceptable responses in open-domain dialogues can be diverse \cite{sato-etal-2017-modeling}.

To directly capture the diversity of outputs in open-domain dialogues, \citeA{sordoni-etal-2015-neural} attempted to collect multiple reference responses from dialogue logs for each test utterance-response pair.
Because automatically-augmented reference responses are noisy, \citeA{galley-etal-2015-deltableu} improved this method by manually rating the augmented reference responses and used the ratings to perform discriminative \bleu evaluation, as detailed later in \S~\ref{sec:delta-bleu}.
\citeA{gupta-etal-2019-investigating} created multiple reference responses by hand for the DailyDialog dataset \cite{li-etal-2017-dailydialog} to confirm the effect of multiple reference responses for evaluating dialogue systems. 
Although these studies confirmed that human-curated reference responses improve the correlation with human judgments, it is costly to create such evaluation datasets for various domains.

In this study, motivated by the moderate correlation with human judgments obtained using multiple human-curated references, we designed a fully automatic uncertainty-aware evaluation method for dialogue systems. 
We confirmed that the embedding-based metric, \bertscore, can take advantage of multiple references that are retrieved and rated by our method.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Reference-free metrics}

Because reference-based metrics with single reference responses do not correlate well with human judgments, various researchers have explored methods to evaluate dialogue systems without relying on reference responses \cite{Tao_Mou_Zhao_Yan_2018,ghazarian-etal-2019-better,mehri-eskenazi-2020-unsupervised,mehri-eskenazi-2020-usr,sinha-etal-2020-learning,sai-etal-2020-improving}. 
The referenced-free methods learn to model the relevance between an utterance and a response in a supervised or unsupervised manner.

\ruber\ \cite{Tao_Mou_Zhao_Yan_2018} is an automatic evaluation method that combines two approaches: its referenced scorer evaluates the similarity between a reference and a generated response using the cosine similarity of their vector representations, whereas its unreferenced scorer, trained by negative sampling, evaluates the relevance between an input utterance and a generated response.
\citeA{ghazarian-etal-2019-better} demonstrated that contextualized word embeddings \cite{devlin-etal-2019-bert} improve the unreferenced scorer but not the referenced scorer in \ruber.
The referenced scorer is similar to \bleu in that they both are referenced-based evaluation metrics.

Some researchers exploited the language recognition ability of large-scale pre-trained models in the evaluation of open-domain dialogue systems.
USL-H \cite{phy-etal-2020-deconstruct} and USR \cite{mehri-eskenazi-2020-usr} evaluate the fluency of generated responses from the probabilities of token output by transforming the masked language model task of \bert and the appropriateness of system responses to the context by transforming the next sequence prediction task of \bert.
Deep AM-FM \cite{Zhang2021} uses the perplexity of the text generation-based pre-trained language model as the evaluation score of the generated responses.
FED \cite{mehri-eskenazi-2020-unsupervised} calculates the perplexity of a given follow-up utterance such as ``Wow! Very interesting'' to the generated responses as an evaluation score.

\yn{The above reference-free metrics are orthogonal efforts to our uncertainty-aware method based on automatically-retrieved and rated reference responses. 
We evaluated \ruber using our method as a referenced scorer to show the utility to combine reference-free metrics with reference-based metrics with multiple references rated by a neural network (\S~\ref{sec:experiment-comparisons}).}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Preliminaries} \label{sec:prel}

Here, we review \dbleu\ \cite{galley-etal-2015-deltableu}, a human-aided evaluation method for text generation tasks with uncertain outputs, after explaining the underlying metric, \bleu\ \cite{papineni-etal-2002-bleu}. 
We then explain the embedding-based metric \bertscore\ \cite{Zhang_2020BERTScore} and \ruber\ \cite{Tao_Mou_Zhao_Yan_2018} that utilizes reference-free metric because each of metric is combined with our method later in \S~\ref{sec:extension} and \S~\ref{sec:experiment-comparisons}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{BLEU}

\bleu\ \cite{papineni-etal-2002-bleu} calculates an evaluation score based on the number of occurrences of $n$-gram tokens that appear in both reference response $r$ and generated response $h$.
Specifically, the score is calculated from a modified $n$-gram precision $p_n$ and a brevity penalty \textsc{bp}
\begin{align}
    \textsc{bleu} & = \textsc{bp} \cdot \exp \left(\sum_{n} \frac{1}{N}\log p_{n}\right),\label{bleu-score}\\
    \textsc{bp} &= \left\{
        \begin{array}
            {ll}{1} & { {\rm if}\  \eta>\rho} \\ {e^{(1-\rho / \eta)}} & { {\rm otherwise} }
        \end{array}
        \right. ,\\
    p_n &=\frac{\displaystyle\sum_i \sum_{g \in n\mathchar`-{\rm grams} (h_i)} \max_j \{\#_{g} (h_i, r_{i, j})\}}{\displaystyle\sum_i \sum_{g \in n\mathchar`-{\rm grams} (h_i)} \#_g (h_i)}.\label{bleu-pn}
\end{align}
Here, $\rho$ and $\eta$ are the average lengths of the reference and generated responses, respectively; $n$ and $N$ are the $n$-gram length and its maximum, respectively; $h_i$ and $\{r_{i,j}\}$ are the generated response and the $j$~th reference response\footnote{\bleu allows multiple reference responses for a single utterance (cf.\ \S~2.2.1 in \citeA{papineni-etal-2002-bleu}).} for the $i$~th utterance, respectively; $\#_{g}(u)$ is the number of occurrences of $n$-gram token $g$ in sentence $u$; $\#_{g}(u,v)$ is defined as 
$\min\{\#_{g}(u),\#_{g}(v)\}$.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{$\Delta$BLEU: Discriminative BLEU} \label{sec:delta-bleu}

\dbleu\ \cite{galley-etal-2015-deltableu} is a human-aided evaluation method for text generation tasks with uncertain outputs, such as response generation in open-domain dialogues.
To augment the reference responses for each test example (an utterance-response pair), following the work by \citeA{sordoni-etal-2015-neural}, \dbleu first retrieves utterance-response pairs similar to the given pair from conversation logs in SNS such as Twitter.
They compute the similarity between the test sample and each utterance-response pair in the retrieval pool by multiplying the similarity between the utterances and the similarity between the responses, each of which is computed by \textsc{bm25} \cite{Robertson1994OkapiAT}.
Next, the responses of the top 15 similar utterance-response pairs and the utterance itself (as a parrot return) are combined with the original response to form an extended set of reference responses.
Each of the extended references is then rated by humans, in terms of its appropriateness as a response to the given utterance. 
Finally, \dbleu calculates $p_n$ (Eq.~\ref{bleu-pn}) with the extended reference $r_{i,j}$ and its manual quality judgment $w_{i,j}$ for the input utterance $i$:
\begin{equation}
    p_n = \frac{\displaystyle\sum_{i} \sum_{g \in n\mathchar`-{\rm grams} (h_{i})} \max _{j: g \in r_{i, j}} \{w_{i, j} \cdot \#_{g} (h_{i}, r_{i, j})\}} {\displaystyle\sum_{i} \sum_{g \in n\mathchar`-{\rm grams} (h_{i})} \max_{j}\{w_{i, j} \cdot \#_g (h_{i})\}}.\label{eq:dbleu}
\end{equation}
In this way, \dbleu weights the number of occurrences of $n$-gram $g$ in Eq.~\ref{bleu-pn} with manual quality judgment $w_{i,j}$.

The problem with \dbleu is the cost of human judgments.
Although we want to evaluate open-domain dialogue systems in various domains, the annotation cost prevents effective evaluation.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{BERTScore}\label{sec:bertscore}

\bertscore\ \cite{Zhang_2020BERTScore} is a reference-based evaluation method for text generation tasks that performs evaluation calculations by soft matching of contextual word embeddings of \bert\ \cite{devlin-etal-2019-bert}.
\bertscore uses a pre-trained \bert and this allows domain- and task-independent evaluation.
Given a reference sentence $r$ with the tokens of $ r_1, \ldots, r_k$ and a candidate sentence $h$ with the tokens of $h_1, \ldots, h_l$, \bertscore first computes \textsc{bert} embeddings of the tokens ($\langle \mathbf{r}_1, $\ldots$, \mathbf{r}_k \rangle$ and $\langle \mathbf{h}_1, \ldots, \mathbf{h}_l \rangle$) by inputting these sentences independently into \bert.
Thereafter, \bertscore obtains the evaluation scores according to the following formula:
\begin{align}\label{eq:pbert}
 P_{\mathrm{BERT}} &=\frac{1}{|h|} \sum_{h_j \in h} \max _{r_i \in r} \mathbf{r}_i^{\top} \mathbf{h}_j,\\\label{eq:rbert}
 R_{\mathrm{BERT}} &=\frac{1}{|r|} \sum_{r_i \in r} \max _{h_j \in h} \mathbf{r}_i^{\top} \mathbf{h}_j,\\
  F_{\mathrm{BERT}}& =\frac{2 \cdot P_{\mathrm{BERT}} \cdot R_{\mathrm{BERT}}}{P_{\mathrm{BERT}} + R_{\mathrm{BERT}}}\;.\label{eq:fbert}
\end{align}
If multiple reference responses are available, \bertscore adopts the highest evaluation score after calculating the scores using each reference response.
The use of multiple references is mentioned in the evaluation task of image caption generation in the original paper (cf. \S~4 in \citeA{Zhang_2020BERTScore}).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{RUBER}\label{sec:ruber}

\ruber\ \cite{Tao_Mou_Zhao_Yan_2018} is an automatic evaluation method for open-domain dialogue systems that combines two types of scorers: 
an unreferenced scorer, which calculates a relevance score between the given utterance and the system response, and a referenced scorer, which evaluates similarity between the system responses and the reference response.

The unreferenced scorer is implemented with a neural network model that takes a generated response $h$ and the query utterance $q$, and returns a score $s_{U}$ by the following process:
first, the generated response $h$ and the quey utterance $q$ are independently vectorized into ${\bf{V}}_h$ and ${\bf{V}}_q$ using bi-directional \textsc{gru} (\textsc{b}i-\textsc{gru}) (Eq.~\ref{eq:ruber-vector}). Thereafter, a quadratic feature $m$ is obtained from ${\bf{V}}_h$ and ${\bf{V}}_q$ using a learnable parameter matrix ${\bf{M}}$ (Eq.~\ref{eq:ruber-mix}).
Finally, multi-layer perceptron (MLP) with a sigmoid function is fed to the joint vector $[{\bf{V}}_h;m;{\bf{V}}_q]$ and outputs a score $s_{U}$ (Eq.~\ref{eq:ruber-mlp}).
\begin{gather}
    {\bf{V}}_h = \text{Bi-GRU}(h),\ {\bf{V}}_q = \text{Bi-GRU}(q),\label{eq:ruber-vector} \\ 
    m = {\bf{V}}_h \cdot {\bf{M}} \cdot {\bf{V}}_q, \label{eq:ruber-mix}\\
    s_{U} = \sigma(\text{MLP}([{\bf{V}}_h;m;{\bf{V}}_q])). \label{eq:ruber-mlp}
\end{gather}
The model is optimized with margin loss $J$ using negative sampling with margin $\Delta$.
\[
    J = \text{max}(0, \Delta - s_U(q,r) + s_U(q,r^{-})),
\]
where $\Delta$ is a hyper-parameter and $r^{-}$ denotes a randomly selected response.


The referenced scorer, $s_R$, is an evaluation metric derived from the cosine similarity of sentence embeddings between generated response $h$ and reference response $r$.
In this metric, a sentence embedding ${\bf{v}}_X$ of sentence $X$ is obtained from the concatenation of the maximum and minimum pooling for the word embeddings \{${\bf{w}}_0,\dots,{\bf{w}}_n$\} of $X$.
\begin{align*}
    {\bf{v}}_\text{max}[i] &= \text{max}({\bf{w}}_0[i],\dots, {\bf{w}}_n[i]),\\
    {\bf{v}}_\text{min}[i] &= \text{min}({\bf{w}}_0[i],\dots, {\bf{w}}_n[i]),\\
    {\bf{v}}_X &= [{\bf{v}}_\text{max};{\bf{v}}_\text{min}],
\end{align*}
where [\textbullet] indexes a dimension of an embedding.


\ruber normalizes scores $s_U(q,h)$ and $s_R(r,h)$ (for a generated response $h$ to the query utterance $q$ and the reference response $r$) to the range of $[0,1]$ before summarizing them to obtain summary score $s(q,r,h)$.
The normalization for each score is given by 
\[
    \Tilde{s}_*=\frac{s_*-\text{max}(S)}{\text{min}(S)-\text{max}(S)}
\]
where $s_*$ means either $s_U$ or $s_R$, and $S$ means the set of scores for each metric in the evaluation sample set.
Summary score $s(q,r,h)$ for the normalized scores ($\Tilde{s}_U(q,h)$ and $\Tilde{s}_R(r,h)$) are obtained using maximum or minimum pooling, or arithmetic or geometric mean. 
In this study, the arithmetic mean was employed because the result on the original paper is the most stable.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Uncertainty-aware Evaluation Method for Dialogue Systems} \label{sec:method}

Here, we describe our approach to the problem of \dbleu described in \S~\ref{sec:delta-bleu}.
To remove the cost of human judgments of extended references, we propose using a neural network trained on automatically collected training data to rate each of the retrieved responses (Figure~\ref{fig:classifier}, \S~\ref{sec:classifier}).
In addition, to diversify the extended reference responses in terms of content and style, we propose a relaxed response retrieval approach using continuous vector representations of utterances only (\S~\ref{sec:collection}).
Additionally, we introduce an extension of our method that uses a pre-trained language model, which is effective for various classification tasks (\S~\ref{sec:extension}).
In this part, we aim to improve the entire model by modifying the proposed method and \dbleu's weighted \bleu to a method using pre-trained language models.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1
\begin{figure}[t]
\begin{center}
\includegraphics{30-2ia10f1.pdf}
\end{center}
\hangcaption{Overview of \ubleu: retrieving diverse reference responses from dialogue logs using utterance U$_1$ only (\S~\ref{sec:collection}) to augment the reference response R$_1$ in each test example, validating their quality by neural network (\textsc{nn})-rater (\S~\ref{sec:classifier}) and evaluating the responses generated by the dialogue system using \dbleu.}
\label{fig:classifier}
\vspace{-0.5\Cvs}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Retrieving diverse reference responses} \label{sec:collection}

Given an utterance-response pair (test example), \dbleu expands the original reference response by retrieving utterance-response pairs, in which both the utterance and response are similar to the test example, from massive dialogue logs (here, Twitter)\@.
Because using the similarity between responses prevents us from retrieving diverse responses in terms of content, we propose considering only the similarity between the utterances. 
In addition, we use an embedding-based similarity instead of \textsc{bm25} to flexibly retrieve semantically-similar responses with synonymous expressions (style variants).

We compute the similarity of utterances using the cosine similarity between utterance vectors obtained from the average of pre-trained embeddings of the words in the utterances.
In addition to the retrieved responses, we add the utterance (as a parrot return) to the reference responses as in \dbleu. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Rating extended reference responses} \label{sec:classifier}

\dbleu manually evaluates the appropriateness of the extended reference responses for the utterance.
To remove this human intervention, we propose rating each reference response using a neural network that outputs a probability for that response as a response to the given utterance.

Specifically, our neural network (\textsc{nn})-rater takes two utterance-response pairs as inputs: a given pair of utterance ${\rm U_1}$ and reference response ${\rm R_1}$ (test example), and a retrieved pair of utterance ${\rm U_2}$ and response ${\rm R_2}$ (Figure~\ref{fig:nn-rater}).
The \nnrater is trained to output the probability that the retrieved response ${\rm R_2}$ for ${\rm U_2}$ can be a response to given utterance ${\rm U_1}$ with response ${\rm R_1}$.
This probability is then used as a quality judgment after normalization to the interval $[-1,1]$ as in \dbleu.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 2
\begin{figure}[t]
\begin{center}
\includegraphics{30-2ia10f2.pdf}
\end{center}
\hangcaption{Overview of \nnrater\ (\S~\ref{sec:classifier}): calculating the quality of response R$_2$ to utterance U$_1$, inputting a triplet of a pair of an utterance U$_1$ and a response R$_1$ of some dialogue and a response R$_2$ of other dialogue into neural network model of \textsc{b}i-\textsc{gru} and \textsc{ffnn}. In the dialogue on estimation and the dialogue of negative label in training data, U${_1}$ and U${_2}$ are different as shown in this figure; however, the U${_1}$ and U${_2}$ of the dialogue of positive label in training data are the same, unlike this figure.}
\label{fig:nn-rater}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The key issue here is how to prepare the training data for the \nnrater.
We used utterances with multiple responses in dialogue data (here, Twitter) as positive examples; we randomly sampled two utterance-response pairs for negative examples.

We then trained the \nnrater in Figure~\ref{fig:nn-rater} from the collected training data.
Because the utterances in the two utterance-response pairs in a positive example are identical while those in a negative example are independent, we do not feed both utterances to the \nnrater.
This input design prevents overfitting.

We constructed \nnrater as a simple model that takes three sentences, concatenates them after vectorization, and outputs the probability of a label.
Specifically, given a test example of utterance ${\rm U_1}$ and response ${\rm R_1}$ and a retrieved utterance-response pair of ${\rm U_2}$ and ${\rm R_2}$, we give two triplets, $\langle {\rm U_1}$, ${\rm R_1}$, ${\rm R_2}\rangle$ and $\langle {\rm U_2}$, ${\rm R_2}$, ${\rm R_1}\rangle$, as inputs to the \nnrater.
Next, we made two vectors by concatenating triplet vectors returned from the bi-directional gated recurrent unit (\textsc{b}i-\textsc{gru}) \cite{cho-etal-2014-properties} as the last hidden state for the utterance and the two responses.
We concatenated forward and backward hidden states ($h_f, h_b$) in the \textsc{b}i-\textsc{gru} to represent an utterance/response vector as $v=[h_f, h_b]$.
We then feed each triplet vector to a feed-forward neural network (\textsc{ffnn}) with softmax function to obtain a pair of probabilities that ${\rm R_2}$ can be a response to ${\rm U_1}$ or not (similarity, another pair of probabilities that ${\rm R_1}$ can be a response to ${\rm U_2}$ or not).
The maximum of these two probabilities is used as the qualitative judgment of the response ${\rm R_2}$ (or ${\rm R_1}$) and multiplied by $-1$ if classified as negative to normalize into $[-1,1]$.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{$\upsilon$BERTScore: Extending $\upsilon$BLEU with BERT} \label{sec:extension}

Recently, large-scale pre-trained models such as \bert\ \cite{devlin-etal-2019-bert} are used in various NLP tasks as-is or as foundation models and outperform existing methods.
Hence, as an extension of \ubleu above, we propose $\upsilon$\bertscore, which employs pre-trained models at each step.
We used \bert to capture meanings of utterances accurately in retrieving and rating extended references and use \bertscore\ \cite{Zhang_2020BERTScore} instead of \bleu to perform soft matching between generated responses and the extended references.
We explain how to extend \ubleu using \bert below.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Dialogue retrieval with contextualized embeddings computed by BERT}
First, instead of using a pre-trained word embedding to collect similar dialogues to a query (\S~\ref{sec:collection}), we adopted the output of each token from \bert as a word embedding. 
Because \bert is trained as a language model, outputs of each token from \bert are contextualized to input sentence compared to a pre-trained word embedding such as GloVe \cite{pennington2014glove}.
Therefore, retrieval with the output from \bert may recognize phrases and style, as well as word meanings.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{BERT as NN-rater}
Second, in the validation for extended reference responses in \ubleu, we used \bert as an implementation of \nnrater instead of \textsc{b}i-\textsc{gru} and \textsc{ffnn}.
As we finetune \bert, which is pre-trained as a language model in a large corpus, as a validator, we can expect high classification performance based on the high language recognition ability.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{BERTScore with weighted reference responses}
Finally, we utilized \bertscore, which performs soft matching between system outputs and reference outputs using contextualized word embeddings, instead of \bleu, which is based on word overlaps.
To weight each extended reference response by its appropriateness score in the \bertscore, we changed the calculation method (Eq.~\ref{eq:pbert}, \ref{eq:rbert}) to the following method with the validity score $w$ to reference sentence $r$.
\begin{align}
 P_{\mathrm{BERT}} & =\frac{1}{|h|} \sum_{h_j \in h} \max _{r_i \in r} w \mathbf{r}_i^{\top} \mathbf{h}_j, \\
 R_{\mathrm{BERT}} & =\frac{1}{|r|} \sum_{r_i \in r} \max _{h_j \in h} w \mathbf{r}_i^{\top} \mathbf{h}_j.   
\end{align}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experimental settings} \label{sec:exp}

Here, we describe how to evaluate our methods for evaluating open-domain dialogue systems.
Using utterances from Twitter (\S~\ref{sec:data}), responses written by humans, and responses obtained by dialogue systems (\S~\ref{ssec:generator}), we evaluated our methods in terms of their correlation with human judgment (Table 1, \S~\ref{sec:setting-classifier}--\ref{sec:experiment-comparisons}).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[b]
\input{10table01.tex}%
\caption{Summary of each multi-reference-based evaluation method compared in the experiments.}
\label{tb:summary-method}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Twitter dialogue datasets} \label{sec:data}

We developed a large Japanese dialogue dataset from the Twitter archive that we have collected since March 2011. 
This archive consists of regularly collected posts by tracked users through the Twitter API\@.\footnote{\url{https://developer.twitter.com/en/docs/twitter-api}}
We initially selected approximately 30 well-known Japanese users to be tracked, and by adding users mentioned or retweeted by these tracked users to be tracked, the number of users has expanded to approximately 2.5 million.
To create a dataset from this archive, we extracted only Japanese-language posts, pre-processed the posts to denoise them, and then obtained conversation logs.

To extract Japanese language posts, we used the language labels provided by the API and ldig,\footnote{\url{https://github.com/shuyo/ldig}} a language classification model specifically designed for Twitter, to increase reliability.
ldig can classify the language of posts on Twitter with more than 99\% accuracy for 17 languages.
Posts classified as Japanese by both models were used from the archive.

To denoise posts, non-linguistic information such as URLs and emojis were removed from the posts.
We also removed or omitted SNS-specific text, such as user names (e.g., @user\_name) and repeated letters or words.
Finally, we removed posts that appear to be from bots or have little content, such as posts that contain less than 5 characters, do not contain more than 60\% Japanese, or are duplicates on the same day.

Finally, we constructed a conversational dataset from the denoised posts.
Specifically, we considered posts that were neither retweets nor references to other posts as utterances and posts that referred to those posts as responses.

We used this dataset for training and testing dialogue systems and for training the \nnrater that evaluates the quality of retrieved responses.
In these experiments, to simulate evaluating dialogue systems trained with dialogue data that are unseen by evaluation methods, we used dialogue data posted during 2017 for training and running the \nnrater, and dialogue data posted during 2018 for training and during 2019 for testing the dialogue systems, as summarized in Table~\ref{tb:dataset}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 2
\begin{table}[t]
\input{10table02.tex}%
\hangcaption{Statistics of the dialogue data used to run each model (component). The numbers in the parentheses denote the year.}
\label{tb:dataset}
\vspace{-1\Cvs}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Target responses for evaluation} \label{ssec:generator}

Following \citeA{liu-etal-2016-evaluate} and \citeA{lowe-etal-2017-towards}, 
we adopted three methods to obtain responses for each utterance in the test set to evaluate various types of dialogue systems:
a retrieval-based method \textsc{c-tfidf} \cite{liu-etal-2016-evaluate} with \textsc{bm25} as the similarity function (\textsc{c-bm25}), a generation-based method \textsc{vhred} \cite{Serban_Sordoni_Lowe_Charlin_Pineau_Courville_Bengio_2017}, and \textsc{human} responses, which are the actual responses except for the reference response, as an ideal response generated by a dialogue system.

Following \citeA{ritter-etal-2010-unsupervised} and \citeA{6163953}, to extract various dialogues from Twitter as training data for the above methods, we recursively follow replies from each non-reply post to obtain a dialogue between two users that consists of at least three posts. 
We then randomly selected pairs of the first utterances and their replies in the obtained dialogues as our dialogue data: $2.4$M pairs for training the \textsc{vhred} and for retrieving responses in \textsc{c-bm25}, $10$K pairs as validation data for the \textsc{vhred}, and 100 pairs as test data.\footnote{To obtain \textsc{human} responses for evaluation, we only used dialogues whose first utterances had more than one response.}
These dialogues were tokenized with SentencePiece \cite{kudo-richardson-2018-sentencepiece}\footnote{The model was trained with a 16,000 vocabulary size on the training dataset.} for \textsc{vhred} and with MeCab 0.996 (ipadic 2.7.0)\footnote{\url{https://taku910.github.io/mecab/}} for \textsc{c-bm25} to retrieve responses based on words that are less ambiguous than subwords.

Finally, six Japanese native speakers in our research group, who are not related to this research project, evaluated the randomly shuffled 300 target responses for the 100 test examples in terms of the appropriateness as a response to a given utterance.
We used a 5-point Likert-type scale, with 1 denoting inappropriate or unrecognizable and 5 denoting very appropriate or seeming to be an actual response.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Response retrieval and scoring}\label{ssec:scoring}

Following \citeA{galley-etal-2015-deltableu}, for each test example, the 15 most similar utterance-response pairs were retrieved to augment the reference response in addition to the utterance itself (as a parrot return) to apply \dbleu, \ubleu, and \ubertscore.
We retrieved utterance-response pairs from approximately $16$M utterance-response pairs of our dialogue data (Table~\ref{tb:dataset}).
These dialogue data were tokenized with MeCab for response retrieval;
we then trained GloVe embeddings \cite{pennington2014glove} to compute utterance or response vectors (\S~\ref{sec:collection}) and performed domain-adaptive pre-training (DAPT) \cite{gururangan-etal-2020-dont} of a pre-trained \bert\footnote{\url{https://huggingface.co/cl-tohoku/bert-base-japanese-whole-word-masking}\label{bert-url}} on this dialogue data.
We used the obtained domain-adapted pre-trained \bert for \ubertscore to retrieve extended reference responses and to build a \bert-based \nnrater as explained in the section below.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{NN-rater to evaluate reference responses} \label{sec:setting-classifier}

To train the \nnrater for evaluating the extended references (\S~\ref{sec:classifier}), we randomly extracted $5.6$M and $10$K utterance-response pairs for training and validation data, respectively. 
The number of positive and negative examples was set equal in both data.
Before these examples were fed to the \nnrater in \ubleu, they are tokenized with SentencePiece\@.

For the \nnrater in \ubleu, we used a $512$-dimensional embedding layer, one \textsc{b}i-\textsc{gru} layer with $512$-dimensional hidden units, five layers for the \textsc{ffnn} with $1024$-dimensional hidden units, and a ReLU as the activation function.
We used the Adam optimizer \cite{DBLP:journals/corr/KingmaB14} with an initial learning rate of 0.001 and calculated the loss by the cross entropy.
We trained the \nnrater with a batch size of 1,000 and up to 15 epochs.
The model with parameters that achieved the minimum loss on the validation data was used for evaluating the test data. 

For the \nnrater in \ubertscore, we used the same architecture as the original \bert base model \cite{devlin-etal-2019-bert}; $12$ layers, $768$ dimensions of hidden states, and $12$ attention heads.
This model was pre-trained on Japanese Wikipedia as of September 1, 2019, and the text file used for training is 2.6GB in size, consisting of approximately 17M sentences.
%%%%See the link\footref{bert-url} for the other details.
    See the link for the other details.
To obtain a domain-adapted pre-trained \bert, we trained the pre-trained \bert again using the training dataset for \nnrater.
In addition, this domain-adapted \bert was trained with the same settings as above to obtain \nnrater.

We then evaluated the appropriateness of each retrieved reference response by humans for \dbleu and by the {\nnrater}s for \ubleu and $\upsilon$\bertscore in terms of appropriateness as a response to a given utterance.
We asked four of the six Japanese native speakers to evaluate the quality of each retrieved reference response.\footnote{We asked only part of them because of the large annotation size and hence the high cost.}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Compared evaluation methods for open-domain dialogue systems} \label{sec:experiment-comparisons}

To observe the impact of two modifications on \dbleu, namely more diverse reference retrieval (\S~\ref{sec:collection}) and automatic reference quality judgment (\S~\ref{sec:classifier}), we first compared \bleu with various reference retrieval methods.
A \bert-based reference retrieval method that uses contextualized embedding from \bert to retrieve dialogues based only on the cosine similarity of utterances is also compared.
Thereafter, we compared \bleu and \bertscore with only one reference, \dbleu, and \ubleu, and $\upsilon$\bertscore.
We then compared our proposed methods, \ubleu and $\upsilon$\bertscore, with \ruber, which uses a reference-free metric, and examined the performance of \ruber when its referenced scorer was replaced with our proposed methods. 

Specifically, we applied each evaluation method to the 300 responses (\S~\ref{ssec:generator}).
\dbleu, \ubleu, and \ubertscore used the extended references of each method in the evaluation.
\bleu used the original (single) references or the extended references.
The referenced scorer in \ruber used the original (single) references with GloVe embeddings,\footnote{We changed a word embedding from word2vec \cite{DBLP:journals/corr/abs-1301-3781}, which was used in the experiments in the original paper \cite{Tao_Mou_Zhao_Yan_2018}. We confirmed that this change improves the performance of the referenced scorer in our experiment.} as 
described in \S~\ref{ssec:scoring}

We evaluated the performance of the evaluation methods in terms of their correlation to human judgments on the 300 responses.
To calculate the correlation, we used Spearman's $\rho$ and Pearson's $r$.
In addition to the correlations with the averaged human judgments, we computed the maximum and minimum correlation with human judgments given by each annotator to understand the stability of the evaluation.
All evaluation methods using the modified $n$-gram precision were calculated with $n\leq2$ (\bleu-2), following \cite{galley-etal-2015-deltableu}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Results} \label{sec:res}

Here, we present the results of the experiments, from the modifications to \dbleu to the combination with \ruber and other analyses.
We first present a comparison of the reference retrieval methods in terms of the performance of \bleu using multiple references extended by each method.
Subsequently, we present a comparison of reference-based metrics including \dbleu, \ubleu, and \ubertscore.
We then demonstrate the performance of the reference-based metrics combined with \textsc{ruber}('s unreferenced scorer, which is a reference-free metric).
We finally analyze the performance of the \nnrater in terms of agreement with the human evaluation and qualitative analysis of each method.

Table~\ref{tb:correlation-collection} lists the correlations between the human judgments and \bleu scores for each reference retrieval method.
We can observe that cosine similarity on embeddings correlates with human judgments more than \textsc{bm25}.
When cosine similarity on embeddings is used, using only the utterances as keys outperformed using both utterance and response.
When we compare the single and multiple reference-based metrics, for Pearson's $r$, only the proposed retrieval method, which uses an embedding-based similarity to utterances, showed a higher minimum correlation than \bleu did with only one reference.
These results confirmed that the proposed retrieval methods were effective for extending the reference responses.
Additionally, a comparison of embedding types confirmed that the \bert-based extension was effective for dialogue retrieval.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[b]
\input{10table03.tex}%
\hangcaption{Correlation between human judgment and \textsc{bleu} with a single or extended reference responses; the extended reference responses are obtained by chaining the key (for utterance-response pairs) to compute similarities and similarity functions.}
\label{tb:correlation-collection}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Table~\ref{tb:correlation-ref} compares the evaluation methods based on reference-based metrics. 
First, we established that the single-reference metrics (\bertscore and the referenced scorer in \ruber) did not correlate well with human judgments, as did single-reference \bleu in Table~\ref{tb:correlation-collection}.
Conversely, multi-reference metrics \dbleu, \ubleu, and \ubertscore correlated more than those.
The comparison between \ubleu and \bleu in Table~\ref{tb:correlation-collection} revealed that our \nnrater improved the minimum correlation with human judgment.
Here, \ubleu was comparable to \dbleu, which implies that our method can successfully automate \dbleu, a human-aided, uncertainty-aware evaluation method.
We observed the highest minimum correlations with human judgments by \dbleu on Spearman's $\rho$ and \ubleu on Pearson's $r$.
This suggests that these metrics are stable evaluations that are easy to agree with various people. 
Conversely, the highest correlation with averaged human judgments by \ubertscore indicated that this metric is the best standard evaluation method.
The comparison between \bertscore and \ubertscore also demonstrated that our reference augmentation is effective for methods other than \bleu.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 4
\begin{table}[t]
\input{10table04.tex}%
\hangcaption{Correlation between reference-based method and human judgment; human refers to the inter-rater correlations.
\ruber (Referenced Scorer) and \bertscore use the original single reference responses, whereas \dbleu, \ubleu, and \ubertscore use the extended reference responses with automatic ratings by each method.}
\label{tb:correlation-ref}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Finally, Table~\ref{tb:correlation-ruber} presents the results of a reference-free metric (the unreferenced scorer in \ruber) and combination methods of reference-based and reference-free metrics as an automatic evaluation method.
The original \ruber combination performed worse than the (reference-free) unreferenced scorer owing to the poor performance of the (reference-based) referenced scorer (c.f. Table~\ref{tb:correlation-ref}).
Although this result is not consistent with the results reported in the original paper \cite{Tao_Mou_Zhao_Yan_2018}, \citeA{ghazarian-etal-2019-better} have failed to reproduce their results.
By replacing the referenced scorer of \ruber with our proposed reference-based automatic methods, we obtained better overall correlations and the best performance with \ubertscore.
This confirms that our uncertainty-aware reference-based metrics are also effective when combined with reference-free metrics.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 5
\begin{table}[t]
\input{10table05.tex}%
\hangcaption{Correlation of human judgment with reference-free method or combination with reference-based method; human refers to the inter-rater correlations. 
\ubleu and \ubertscore use the extended reference responses with automatic ratings by each method.}
\label{tb:correlation-ruber}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Analysis: Evaluation of NN-rater}
In the proposed methods (\ubleu and \ubertscore), the reference response augmentation method and validation method to the retrieved responses were changed from \dbleu and \bertscore.
The improvement of the reference response augmentation method was evaluated to be effective in increasing the correlation with human judgments in Table~\ref{tb:correlation-collection}.
However, the extent to which the validator (\nnrater) behaves like humans is not validated in the experiments.
Therefore, we analyzed the correlation between \nnrater's scores for extended reference responses with the human judgments.
As this analysis requires human evaluation costs to measure the correlation with each \nnrater for the responses retrieved by each method, we used the \bert-based \nnrater used in \ubertscore, which had the highest correlation with human judgments in the experiments.
To obtain human judgments of \ubertscore's extended reference responses, we asked four annotators, who are Japanese native speakers in our research group and were not involved with this study, to evaluate the appropriateness of the extended reference responses given the target utterance.
These instructions are identical to the settings in \S~\ref{sec:setting-classifier}; specifically, we used responses of the 15 most similar utterance-response pairs retrieved by \ubertscore for the test data.

Table~\ref{tb:correlation-nn-ratotr} shows the results of the analysis.
First, we observed that the correlations among annotators did not differ from those in Table~\ref{tb:correlation-ref}. 
This implies that the quality of human judgments is almost the same among each other. 
Second, we confirmed that the \bert-based \nnrater had a high correlation with human judgments. 
This may be because \nnrater is trained with human-human dialogues and tested in almost the same situation.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 6
\begin{table}[t]
\input{10table06.tex}%
\hangcaption{Correlation between \bert-based \nnrater and human judgment to extended reference responses using \bert; human refers to the inter-rater correlations.}
\label{tb:correlation-nn-ratotr}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 7
\begin{table}[t]
\input{10table07.tex}%
\hangcaption{Examples of responses retrieved and evaluated by our method for a given test example, along with evaluation scores for responses generated by \textsc{c-bm25}. 
\bleu refers to \bleu score with the original response, whereas \textsc{bleu}$_{\textrm{multi}}$ refers to \bleu score with \ubleu's extended references.
(Note: \textsc{bleu}$_{\textrm{multi}}$ does not take \nnrater scores into account.)
For comparison, we normalized all evaluation scores to the interval for \bleu, i.e., [0, 1].}
\label{tb:examples}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Examples} Table~\ref{tb:examples} shows examples of responses retrieved and evaluated by our method, along with evaluation scores for responses generated by \textsc{c-bm25}.
The \bleu score with a single-reference response was almost zero because few words matched in the reference and generated responses.
Multi-reference \bleu ($\textsc{bleu}_{\textrm{multi}}$) also scored low.
This is because $\textsc{bleu}_{\textrm{multi}}$ uses inappropriate or irrelevant responses as reference responses and is disturbed by them.
Conversely, \dbleu, \ubleu, and \ubertscore gave a slightly higher score to the generated response, similar to the human evaluation\footnote{Note that since evaluation metrics are (meta-)evaluated in correlation with human judgment, it is not possible to decide the metric that is better from this perspective alone.}.
In \dbleu, validation scores are high quality because they are based on human judgment; however, the retrieval method based on exact word matches is poor and often tends to collect inappropriate responses.
In Table~\ref{tb:examples}, the low-rated (second) response (``シンクロ率高すぎでしょ'': ``Synchronization rate is too high'') collected only because it matches (``すぎでしょ'': ``... is too ..'' or ``... is very ..'').
In contrast, because \ubleu searches for responses based solely on utterance similarity, \ubleu's extended reference responses will have a more diverse word distribution than \dbleu's.
Finally, \ubertscore also collects various responses and can utilize a \bertscore-based evaluation metric that is calculated by the similarity of the semantic meaning of words.

\dbleu and our proposed reference augmentation are useful when the reference and the generated response are not similar in an exact match or even word-based sentence similarity as in this example.
In particular, \ubertscore can perform embedding-based flexible evaluation, which captures possible semantically-similar responses, whereas the utterance-based retrieval can collect semantically-dissimilar response candidates.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusions}

Herein, 
\pagebreak
we propose methods to remove the need for costly human judgment in \dbleu\ \cite{galley-etal-2015-deltableu} and obtain automatic uncertainty-aware metrics for dialogue systems. 
Our proposed methods rate diverse reference responses retrieved from massive dialogue logs using a neural network trained with automatically-collected training data and score the generated responses using the retrieved responses and their scores. 
Experimental results on massive Twitter dialogue data revealed that \ubleu is comparable to human-aided \dbleu and is effective if it is combined with reference-free metrics in \ruber.
We also confirmed that \bert-extended \ubleu, \ubertscore, is the best performance in the experiments.
These results confirm the importance of considering the diversity of outputs in evaluating open-domain dialogue systems.

We release all code and datasets (tweet IDs) to promote the reproducibility of our experiments\footnote{\texttt{http://www.tkl.iis.u-tokyo.ac.jp/\%7Etsuta/jnlp}}.
The readers are referred to our code to evaluate their dialogue systems for their native languages while reusing dialogue data used to train the target dialogue systems for evaluation to run \ubleu.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section*{Limitations}

More powerful open-domain dialogue systems such as BlenderBot \cite{roller-etal-2021-recipes} and Meena \cite{https://doi.org/10.48550/arxiv.2001.09977} are publicly available, although in this article we evaluated only VHRED, retrieval-based system, and human as a dialogue system.
Therefore, the effectiveness of the proposed method for these new powerful dialogue systems has not yet been verified and guaranteed.
Because it is difficult for a single research group to perform a comprehensive evaluation of various dialogue systems, our research communities need to adopt automatic metrics along with human judgments to confirm the applicability of the metric.
\citeA{smith-etal-2022-human} reported that whether the differences in performance of a dialogue system (BlenderBot) with different parameters are detectable by human judgment depends on the method of human judgment.
Therefore, even if an automatic evaluation method is replaced by human judgment, it is necessary to discuss what differences can be detected by the evaluation method.
This study merely demonstrates that the proposed method can address the diversity of responses, which is a challenge in open-domain dialogue evaluation research, by augmenting a reference response of the reference-based evaluation metric.
Therefore, what differences in evaluation methods are detectable for various dialogue systems should be carefully discussed as a general issue in open-domain dialogue evaluation research.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment

This work was supported by JST CREST Grant Number JPMJCR19A4, Japan, and JSPS KAKENHI Grant Number 21H03494 and 22H00508.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.7}
\bibliography{10refs}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Yuma Tsuta}{%
He received his B.S. in Integrated Science from the University of Tokyo in 2018, and his M.S. in Information and Communication Engineering from the University of Tokyo in 2020. He is currently working towards a doctoral degree at the University of Tokyo. His research interests include natural language processing, and in particular open-domain dialogue systems.
}

\bioauthor[:]{Naoki Yoshinaga}{%
He received his Ph.D. from the University of Tokyo in 2005. He has been JSPS Research Fellow (DC1, PD) from 2002 to 2008, and Associate Professor at the University of Tokyo since 2016. His research interests include computational linguistics and machine learning, in particular efficient and adaptive natural language processing in the wild.
}

\bioauthor[:]{Masashi Toyoda}{%
He received his Ph.D. degree from the Tokyo Institute of Technology in 1999. He worked as Associate Professor at the University of Tokyo from 2006 to 2018, and has been professor since 2018. His research interests include analysis and interactive visualization of Web, social media, and IoT data.
}
\end{biography}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\biodate


\end{document}

