\documentclass[english]{jnlp_3.3}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{hangcaption_jnlp}
%%%%\usepackage{udline}
%%%%\setulminsep{1.2ex}{0.2ex}
%%%%\let\underline=\ul
\allowdisplaybreaks

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
%%%%\usepackage[dvipdfmx,nosetpagesize]{graphicx}
%%%%\usepackage{amsmath}
%%%%\usepackage{times}
%%%%\usepackage{latexsym}
%%%%\usepackage{tabularx}
%%%%\usepackage{soul}
%%%%\usepackage[utf8]{inputenc}
%%%%\usepackage{xstring}
%%%%\usepackage{url}
\usepackage{multirow}
%%%%\usepackage{CJKutf8}
%%%%\usepackage{minted}
%%%%\newcommand{\secref}[1]{\StrSubstitute{\getrefnumber{#1}}{.}{ }}
%%%%\newcommand{\XXX}[1]{\textcolor{red}{XXX #1}}
%%%%\def\repl#1#2{\textcolor{red}{XXX \sout{#1}}\textcolor{blue}{#2}}
\newcommand*{\enja}{EN$\rightarrow$JA}
\newcommand*{\jaen}{JA$\rightarrow$EN}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{28}
\Number{2}
\Month{June}
\Year{2021}
\Issuetitle{Special Issue: XXXXXXXX}

\received{2020}{10}{28}
\revised{2020}{12}{18}
%%%%\rerevised{2006}{5}{6}
\accepted{2021}{1}{18}

\setcounter{page}{380}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Japanese--English Conversation Parallel Corpus for Promoting Context-aware Machine Translation Research}
\eauthor{Mat{\=\i}ss Rikters\affiref{Author_1} \and Ryokan Ri\affiref{Author_1} \and Tong Li\affiref{Author_1} \and Toshiaki Nakazawa\affiref{Author_1}} 
\eabstract{%
Most machine translation (MT) research has focused on sentences as translation units (sentence-level MT), and has achieved  acceptable translation quality for sentences where cross-sentential context is not required in mainly high-resourced languages. Recently, many researchers have worked on MT models that can consider a cross-sentential context. These models are often called context-aware MT or document-level MT models. Document-level MT is difficult to 1) train with a small amount of document-level data; and 2) evaluate, as the main methods and datasets focus on sentence-level evaluation. To address the first issue, we present a Japanese--English conversation corpus in which the cross-sentential context is available. As for the second issue, we manually identify the main areas where sentence-level MT fails to produce adequate translations in the lack of context. We then create an evaluation set in which these phenomena are annotated to alleviate the automatic evaluation of document-level systems. We train MT models using our corpus to demonstrate how the use of context leads to improvements.
}
\ekeywords{Parallel Corpora, Machine Translation, Manual Annotation}

\headauthor{Rikters et al.}
\headtitle{JA--EN Conversation Parallel Corpus for Promoting Context-aware MT Research}

\affilabel{Author_1}{}{The University of Tokyo}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}
\label{sec:intro}

The quality of machine translation (MT) for written text and monologues has vastly improved because of the increased amount of available parallel corpora that are required to train MT models and recent advances in neural network technologies that enable efficient use of the available corpora. However, there is much scope for improvement in the context of dialogue or conversation translation. One typical case is the translation from a pro-drop language to a non-pro-drop language, where correct pronouns must be supplemented according to the context. The omission of pronouns occurs more frequently in spoken language than in written language. Recently, to solve this type of problem context-aware MT models have attracted considerable research attention \shortcite{tiedemann-scherrer-2017-neural,voita-etal-2019-good}. 

However, there are almost no parallel conversation corpora with context information, except for the noisy Open Subtitles corpus \shortcite{tiedemann-2016-finding}. Most of the available parallel corpora are shuffled, and sentences do not remain in the original order as in the source material. Some of the corpora are automatically filtered to improve quality, which means that even if they were in the original order, there would be gaps between consecutive sentences. Lastly, not all unfiltered and unshuffled corpora have document boundaries. All these factors hinder the training of context-aware models.

Thus, a parallel document- and sentence-aligned conversation corpus would be advantageous in taking MT research in this field to the next stage. In this paper, we introduce a newly constructed Japanese--English conversation corpus that contains three sub-corpora: business scene dialogue (BSD) \shortcite{rikters-etal-2019-designing}, Japanese translation of AMI meeting corpus (AMI\footnote{AMI corpus - \url{http://groups.inf.ed.ac.uk/ami/corpus/index.shtml}}) \shortcite{mccowan2005ami}, and the Japanese translation of OntoNotes 5.0 (ON\footnote{OntoNotes Release 5.0 - \url{https://catalog.ldc.upenn.edu/LDC2013T19}}) \shortcite{ontonotes}. The corpus contains multiperson conversations in various situations: business scenes, meetings under specific themes, broadcast conversations, and telephone conversations. Our main contribution is the production and release of these datasets, their analysis, and markup of errors caused by translating context-dependent sentences using context-agnostic MT systems.

The structure of this paper is as follows: We summarise related work in Section \ref{sec:related} and in Section \ref{sec:about} we explain how the corpus was constructed. In Section \ref{sec:error-analysis}, we investigate the shortcomings of modern MT with regard to translating conversations. Section \ref{sec:mt-exp} provides an overview of our MT experiments, and in Section \ref{sec:human-eval} we present our human evaluation approach and results. Finally, we conclude the paper in Section \ref{sec:conclusion} and introduce plans for future work.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Work}
\label{sec:related}

Modern MT systems are based on neural networks trained with parallel sentences between languages, that is, parallel corpora. Most of the existing ready-to-use parallel corpora comprise written text obtained from Web crawl, patents \shortcite{ntcir9-patmt}, and scientific papers \shortcite{NAKAZAWA16.621}. Even though some parallel corpora do comprise spoken language, they are generally monologues \shortcite{cettoloEtAl:EAMT2012,di-gangi-etal-2019-must} or contain a lot of noise \shortcite{tiedemann-2016-finding,pryzant-etal-2018-jesc}. Most of the MT evaluation campaigns such as WMT\footnote{\url{http://www.statmt.org/wmt20/}} and WAT\footnote{\url{http://lotus.kuee.kyoto-u.ac.jp/WAT/}} adopt the written language, monologue, or noisy dialogue parallel corpora for their translation tasks. Among these corpora, there is only one clean, dialogue parallel corpus \shortcite{salesky2018slt} adopted by IWSLT\footnote{\url{http://workshop2019.iwslt.org}} for the conversational speech translation task. 

JParaCrawl \shortcite{morishita-etal-2020-jparacrawl} is a recently announced large English--Japanese parallel corpus built by crawling the web and aligning parallel sentences. Its size is impressive, but it is composed of noisy web-crawled data and contains many duplicate sentences. Unlike our corpus, JParaCrawl does not have meta-information or access to cross-sentential context. 

Translation that requires contextual information has received much attention in recent years. \citeA{voita-etal-2019-good} evaluated the limitations of modern MT systems when translating English to Russian; they constructed new development and evaluation sets based on human evaluation. The sets target linguistic phenomena: dexis, ellipsis, and lexical cohesion.
Our dataset does not aim to evaluate specific discourse phenomena; however, the development/evaluation sets contain complete documents of consecutive sentences, which allow the analysis of discourse phenomena involved in translation.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[b]
\input{03table01.tex}
\hangcaption{Statistics of the constructed conversation parallel corpus. {\jaen} represents scenarios that are written in Japanese then translated into English and {\enja} represents scenarios constructed in the reverse way.}
\label{tab:total-unique-other}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Corpus Description}
\label{sec:about}

Our corpus consists of three sub-corpora, each of which originates from different sources: BSD, AMI, and ON. The number of sentence pairs from each corpus is shown in Table \ref{tab:total-unique-other}. We regard sentence pairs as unique if at least one of the source and target sentences are different. For example, if we have two identical sources, but they have different translations, the pairs are considered unique. We provide balanced development and evaluation splits from only the BSD sub-corpus, as it is the least noisy part in our corpus.

BSD has been newly constructed, whereas AMI and ON are translations of the existing English versions of these corpora. Even though BSD has the highest quality of the three, we use all three, as it is beneficial to have more data for training. Because AMI and ON have similar domains, they allow a meaningful increase in the total relevant training dataset size for experiments. We also conducted several ablation experiments (end of Section \ref{sec:mt-results}), showing that removing AMI and ON from the training data decreases MT quality.
In the following subsections, we describe the construction process of each sub-corpus and its contents.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Business Scene Dialogue}
\label{sec:about-bsd}

The BSD sub-corpus was constructed without the use of any pre-existing resources. We choose business conversation as the domain of the corpus because the business domain is neither too specific nor too general, and we believe that a clean conversational parallel corpus is useful for opening new directions for MT research. We hope that this corpus will become a standard benchmark dataset for machine translation.

The three steps in the construction process are as follows: 1) selecting business scenes, 2) writing monolingual conversation scenarios according to the selected scenes, and 3) translating the scenarios into the other language, each of which is explained in detail in the following subsections. There were a total of 23 English scenario writers, 28 Japanese scenario writers, 13 {\enja} translators, and 8 {\jaen} translators, along with three overall translation checkers. Some translators were also scenario writers.

This corpus is unique in that each scenario is annotated with scene information, as shown at the top of Table \ref{table:corpus-example}. In conversations, utterances are often very short and vague; therefore, it is possible that utterances will be translated differently depending on the situations in which the conversations take place. For example, the Japanese expression ``すみません'' can be translated into several English expressions such as ``Excuse me.'' (when you call a store attendant); ``Thank you.'' (when you receive gifts); or ``I'm sorry.'' (when you need to apologise). Using the scene information, it is possible to differentiate the translations, which is difficult to do with only contextual sentences. Furthermore, it might be possible to connect scene information to the multimodal translation, such as estimating scenes using visual information, which has rarely been studied.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 2
\begin{table}[t]
\input{03table02.tex}
\hangcaption{An example of the Japanese--English business conversation parallel corpus. Scene: phone call; title: customer inquiry.}
\label{table:corpus-example}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{1) Business Scene Selection}

The business scenes were carefully selected to cover a variety of business situations, including meetings and negotiations, as well as so-called ``water-cooler'' chats. The details are presented in Table \ref{table:overview}. Although most of the scenes are clearly distinct, ``face-to-face'' and ``general chatting'' may seem very similar at first. ``Face-to-face'' represents work-related conversations in the business scene where people talk directly to each other. Conversely, ``general chatting'' refers to more of a ``water-cooler'' conversation between colleagues, which is far less formal and not always business-centred. The ``meeting'' scene represents situations where the conversation is more one-to-all or all-to-all as opposed to ``face-to-face,'' which would generally be one-to-one. We include ``general chatting'' in business scenes because it is useful in situations such as business-related parties.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[t]
\input{03table03.tex}
\caption{Detailed statistics of the BSD sub-corpus.}
\label{table:overview}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We also took care not to select specialised scenes that are suitable only for a limited number of industries. We ensured that all selected scenes were generic to a broad range of industries.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{2) Monolingual Dialogue Scenario Writing}

Dialogue scenarios were written monolingually for each of the selected business scenes. Half the monolingual scenarios were written in Japanese and the other half were written in English (42,400 sentences for each language). This is because we aim to cover a wide range of lexicons and expressions for both languages in the corpus. Writing the scenarios in only one language may not incorporate useful, important expressions in the other language when they are translated in the following step.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{3) Scenario Translation}

The monolingual scenarios were translated into the other language by human translators. They were asked to make the translations not only accurate but also as fluent and natural as possible. This principle is adopted to eliminate several common tendencies of human translators when performing Japanese--English translation on a written text. For example, Japanese pronouns are usually omitted in a dialogue; however, when English sentences are literally translated into Japanese, translators tend to include unnecessary pronouns. It is acceptable as a written text but rather unusual as a spoken text.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Quality Control}

The scenario writing and translation were performed by several scenario writers and several translators; therefore, quality control became necessary. In addition, it is important to check whether the scenarios are suitable for a business scene. To guarantee quality and suitability, we asked one person to supervise the entire construction process and regulate its execution. The person satisfies the following conditions to guarantee that the conversations are natural:
\begin{itemize}
    \item has the experience of being engaged in language learning programs, especially for business conversations
    \item is able to smoothly communicate with others in various business scenes in both Japanese and English
    \item has the experience of being involved in business
\end{itemize}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{AMI Meeting Parallel Corpus}

The original AMI Meeting Corpus is a multimodal dataset containing 100 hours of meeting recordings in English.
The parallel version was constructed by asking professional translators to translate the utterances from the original corpus into Japanese. As the original corpus consists of speech transcripts, the English sentences contain many short utterances ({\it e.g., ``Yeah'' and ``Okay''}) or fillers ({\it e.g., ``Um''}), and these are translated into Japanese as well. Therefore, the AMI sub-corpus contains many duplicates at the sentence level (see Table \ref{tab:total-unique-other}), but with different contexts, making them unique at the document level.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{OntoNotes 5.0 Parallel Corpus}

The original OntoNotes comprised various genres of text (news, telephone speech, weblogs, newsgroups, broadcast, talk shows) in three languages (English, Chinese, and Arabic) with additional annotated information-syntax and predicate argument structure, word sense linked to an ontology and coreference. We extracted the English subsets of broadcast conversations and telephone conversations, and had professional translators translate them into Japanese.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Challenges in Translating Conversations}
\label{sec:error-analysis}

To understand the difficulty of translating conversations, we conducted an analysis of the newly constructed corpus. We especially focus on examples in which context is needed to correctly understand and translate the individual sentences. We analyse the {\jaen} translation results of the BSD corpus and {\enja} translation results of the AMI and ON corpora. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Analysis of the BSD Corpus}
In this subsection, we provide an error analysis of the {\jaen} MT output using the BSD corpus. We choose to use Google Translate,\footnote{\url{https://translate.google.com/} (May 2019)} one of the most powerful publicly available neural machine translation (NMT) systems to produce the translations.


Our primary focus is to understand how many sentences require a context to be properly translated. We randomly sample 10 scenarios (322 sentences) from the corpus, and manually check the translations for critical translation errors, ignoring fluency or minor grammatical mistakes. From the 322 sentences, 12 sentences have errors due to phrase ambiguity that require the context (an example is shown in Figure \ref{fig:phrase-ambiguity}) or the knowledge of real-world situation for a proper translation, and 18 errors due to zero anaphora, which are described in the following section, in the source language (Japanese). Even though they are not that frequent, these types of errors were the only critical errors that we found during our analysis. The more frequent errors were generally minor and did not change the meaning of the translation. Now, we focus on pronoun errors.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1
\begin{figure}[b]
\begin{center}
\includegraphics{28-2ia3f1.pdf}
\end{center}
\hangcaption{An example of translation error due to phrase ambiguity. The word 「お揃い」 is wrongly translated into {\it a match}, which can be resolved by understanding the context, where people are gathered.}
\label{fig:phrase-ambiguity}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Zero Anaphora}
\label{sec:anaphora}

As an important preliminary step, we briefly introduce a grammatical phenomenon called {\it zero anaphora}. In Japanese, some arguments of verbs are often omitted from phrases when they are obvious from the context. When translating these sentences into English, one often has to identify the referent of the omitted argument and recover it in English, as English does not allow the omission of core arguments (i.e., subject, object). In the example in Figure \ref{fig:zero-example}, the subject of the verb 買った is omitted, but in the English translation, a pronoun, for example {\it he}, has to be recovered. Note that the subject could be anyone, not necessarily {\it he}, depending on the context. The task of identifying the referent of zero anaphora is called {\it zero anaphora resolution}, which is one of the most difficult tasks in NLP.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 2
\begin{figure}[b]
\begin{center}
\includegraphics{28-2ia3f2.pdf}
\end{center}
\caption{An example of zero anaphora.}
\label{fig:zero-example}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

In Japanese, zero anaphora is very pervasive, especially in conversation, as the context of the conversation (either topical or visual) helps identify the referent.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Quantitative Analysis}
\label{sec:quantitative-anaphora}

To estimate how many sentences need zero anaphora resolution in the business conversation corpus, we counted the number of sentences with the personal pronouns ({\it e.g.}, 彼, 彼女, 私, あなた in Japanese, {\it I}, {\it you}, {\it he}, {\it she} in English) in both Japanese and English. As a result, 62\% of English sentences contain personal pronouns, whereas only 11\% of Japanese sentences do. This means that about 50\% of the sentences in the corpus potentially need zero reference resolution they are translated from Japanese into English.

To determine what kinds of zero pronouns are difficult to translate, we again heuristically count the number of translation errors of pronouns for the entire corpus. We counted the number of translated sentences that had pronouns different from their reference sentences. Using this heuristic, we detected 3,653 errors (12\% of the whole corpus). The top 10 most frequent errors are shown in Figure \ref{fig:error-graph}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 3
\begin{figure}[t]
\begin{center}
\includegraphics{28-2ia3f3.pdf}
\end{center}
\caption{Top 10 frequent errors of pronoun translation (critical errors denoted in boldface).}
\label{fig:error-graph}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Some errors such as {\it we} $\rightarrow$ {\it I}, {\it I} $\rightarrow$ {\it me} might not be critical and cannot be regarded as translation errors. However, there are still many critical errors among first-, second-, and third-person pronouns (denoted in boldface in the graph).

Looking at the pronouns that the NMT system produced, we can see the tendency of the system to generate frequent pronouns such as {\it you} and {\it I}. This suggests that the current system tries to compensate source (Japanese) zero pronouns simply by generating frequent target (English) pronouns. When the referent is denoted in relatively infrequent pronouns in the target language, it is difficult to translate correctly. To deal with this problem, we need to develop more sophisticated systems that consider the context.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Qualitative Analysis}
\label{sec:qualitative-anaphora}

This section exemplifies some zero-anaphora translation errors and discusses what kind of information is needed to perform the correct translation.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Translation that Needs World Knowledge and Inference}

In Figure \ref{fig:google-error-example1}, the subjects of the verbs are omitted in the source sentence 「（彼は: he）仕事もあまりしない上に、（彼は: he）休み、早退ばかりを希望するから」. This causes the NMT system to incorrectly translate the zero pronouns into {\it I}, although they actually refer to {\it Paul} in the previous sentence and thus have to be translated into {\it he}. Even though there are two persons mentioned in the context sentence, the branch manager and Paul, from the context that Paul is about to be dismissed, it is understandable that Paul is the one who does not work much, takes frequent days off, and often asks to leave early.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 4
\begin{figure}[b]
\vspace{-0.5\Cvs}
\begin{center}
\includegraphics{28-2ia3f4.pdf}
\end{center}
\hangcaption{An example of {\jaen} Google Translate output. The words in boldface denote the same referent (Paul).}
\label{fig:google-error-example1}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Resolving these zero pronouns, however, is not straightforward, even if one has access to information from the previous sentence. For example, to identify the subject of 「仕事もあまりしない」(doesn't work much), one has to know ``laziness can lead to being dismissed'' and thereby infer that Paul, who is about to be dismissed, is the subject. Existing contextual NMT systems \shortcite{voita-etal-2018-context,bawden-etal-2018-evaluating,maruf-etal-2019-selective} still do not seem to be able to handle this complexity.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 5
\begin{figure}[b]
\begin{center}
\includegraphics{28-2ia3f5.pdf}
\end{center}
\hangcaption{An example of {\jaen} Google Translate output. Correct translation requires speaker information.}
\label{fig:google-error-example2}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Translation that Needs to Know Who Is Talking}

In Figure \ref{fig:google-error-example2}, again, the subject is omitted in the source sentence 「（君は: you）もう少しの辛抱だよ。」. The NMT system incorrectly translates zero pronouns into {\it I}.

It is worth noting that the type of zero pronoun differs from that in Figure \ref{fig:google-error-example1} in that the referent in Figure \ref{fig:google-error-example2} does not linguistically appear within the text (called {\it exophora}), whereas the other does ({\it endophora}) \shortcite{brown_yule_1983}. The referent of the zero pronoun in Figure \ref{fig:google-error-example2} is the listener of the utterance ({\it you}), and it usually does not have another linguistic item (such as the person's name) that can be referred to. Although some modality expressions and verb types can provide constraints to the possible referents \shortcite{Nakaiwa:1996vb}, the resolution of exophora essentially requires reference to the situation.

In this case, the correct translation depends on who is speaking. In the original conversation, the utterance is from Speaker 2 to Speaker 1, and given the context, one can infer that Speaker 2 is speaking to console Speaker 1 and thus, the subject should be {\it you} (Speaker 1). However, if the utterance was from Speaker 1, he would then just be complaining about his situation saying, {\it ``{\bf I} just need a bit more patience''}. This example emphasises the fact that speaker information is essentially needed to correctly translate some utterances in a conversation.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Notes on {\enja} Translation}

So far, our discussion has focused on {\jaen} translation. In {\enja} translation, the problem of anaphora usually does not arise; however, the variety of Japanese pronouns poses a problem. One English pronoun usually has many translation candidates in Japanese. For example, the English pronoun {\it you} can be translated into the Japanese pronouns, あなた, お前, 君, and many more. They differ in terms of formality, gender, age, and relative social status of the speaker and audience. For the MT system to correctly translate the pronouns, it needs to understand the speaker's information and the relationship between the speaker and the audience. We leave the resolution of this problem as future work.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Analysis of AMI and ON Corpora}

We manually analyse the AMI and ON corpora in a manner similar to the analysis of the BSD corpus by investigating contextual information requirements for {\enja} MT.
We randomly sampled 200 and 100 sentence pairs from the ON and AMI, respectively. In the case of ON, 50\% of the pairs were from broadcast conversations, and 50\% were from telephone conversations. We translate the sentences using Google Translate\footnote{\url{https://translate.google.com/} (November 2019)} and check the translations for errors, ignoring fluency or minor grammatical mistakes.

Unlike the {\jaen} results for BSD, where more than 50\% of errors were due to zero anaphora, there are mainly two types of causes for errors detected in this analysis: phrase ambiguity and absence of world knowledge. Most of the errors (Table \ref{tab:ami-on-overview}) are caused by phrase ambiguity, for which taking context sentences into account can be considered as a possible solution. Conversely, the transcripts in ON broadcast conversations contain a variety of named entities (e.g., Shia, one of the two main branches of Islam) and abbreviations (e.g., CPC, Communist Party of China). To solve this problem, either domain-specific training data or additional mechanisms that account for world knowledge would be required. Even though phrase ambiguity and world knowledge errors are not frequent, these types of errors were the only critical errors that we found during our analysis. The more frequent errors were generally minor and did not change the meaning of the translation.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 4
\begin{table}[t]
\input{03table04.tex}
\hangcaption{Number of {\enja} translation errors detected in the randomly sampled sentences from AMI and ON corpora.}
\label{tab:ami-on-overview}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Machine Translation Experiments}
\label{sec:mt-exp}

To test our corpus for its intended use case, we trained several configurations of MT systems using only a limited amount of data from the corpus and compared them with systems trained on larger datasets. The total amount of training, development, and evaluation data are summarised in Table \ref{tab:data-table}. WMT is obtained from the WMT20 shared task on MT of news ($\sim$13M parallel sentences mainly consisting of movie subtitles, crawled websites, TED talks and Wikipedia text). It should be noted that WMT is a sentence-level parallel corpus that does not contain document-level information.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 5
\begin{table}[t]
\input{03table05.tex}
\caption{Statistics of the corpora used for the experiments.}
\label{tab:data-table}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Experiment Setup}

For the sentence-level systems, we trained the transformer \shortcite{NIPS2017_7181} models with the \textit{transformer-base} configuration, implemented using the Sockeye \shortcite{hieber-etal-2018-sockeye} library.

To train our context-aware systems, we experimented with two approaches: sentence concatenation \shortcite{tiedemann-scherrer-2017-neural} with source-side factors \shortcite{sennrich2016linguistic-wmt} and context-aware decoder, CADec \shortcite{voita-etal-2019-good}. We used the same toolkit and parameters similar to those in our sentence-level systems for the former and the CADec toolkit with the default parameters for the latter. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Sentence Concatenation with Source-side Factors}

In the concatenation context-aware MT, the previous sentence from the same document is prepended, followed by a beginning of sentence tag \textit{$<$bos$>$}, to the source sentence. In addition, source-side factors are provided to specify whether a token represents context (``C'' in Table  \ref{tab:factors}) or the source sentence to be translated (``S'' in Table \ref{tab:factors}). These factors are converted into embeddings, and each source token embedding is concatenated with the respective factor embedding.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 6
\begin{table}[t]
\input{03table06.tex}
\hangcaption{%
Examples of training data source sentences and the respective source-side factors for the concatenated context-aware experiments. The first sentence in the table has no previous context, as it is the first sentence in the respective document. The second sentence has the first sentence as context, followed by the beginning of the sentence tag \textit{$<$bos$>$}. Each C refers to each context token, and each S - source refers to the sentence token to be translated. We consider the beginning of the sentence token (\textit{$<$bos$>$}) as the context.}
\label{tab:factors}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Context-aware Decoder}

The CADec toolkit performs translation in two steps. First, a normal transformer model produces an initial translation. This model can be trained with normal sentence-level parallel data without context. After that, the context-aware decoder fixes the translation by incorporating the information from contextual sentences of the source and target language. This decoder is trained using document-level parallel data.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Results}
\label{sec:mt-results}

We evaluate automatic translations using the SacreBLEU\footnote{Version string - BLEU+case.mixed+numrefs.1+smooth.exp+tok.13a+version.1.2.21} tool \shortcite{post-2018-call}, which calculates BLEU scores \shortcite{papineni-etal-2002-bleu} by comparing each automatic translation with a human-produced reference. A higher BLEU score indicates a better translation.
The results in the upper part of Table \ref{tab:mt-results} show that decent quality (\textgreater{} $\sim$15 BLEU) MT models can be trained by using only our corpora (AMI, BSD, ON data, and baseline model). For {\jaen} the scores slightly improve by training contextual models (concatenated context and concatenated context + source-side factors), which indicates that there are context-dependent sentences in our evaluation set that benefit from the additional information. However, for {\enja} translation, the scores decreased slightly. We investigate this further by performing human evaluation in Section \ref{sec:human-eval}. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 7
\begin{table}[t]
\input{03table07.tex}
\hangcaption{%
MT experiment results in BLEU scores on our evaluation dataset. The first four rows use only our corpus (all parts: AMI, ON, BSD) for training, but the training methods change. The last four rows use the baseline training method with different training data setups - only WMT20 data and WMT20 along with our corpus (only the BSD part and AMI and ON parts).}
\label{tab:mt-results}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We were unable to find a clear reason why models trained with CADec underperformed, even our baseline. One possible explanation for the underperformance could be that it uses three context sentences at once for each content sentence. These lines of four consecutive sentences do not overlap with the previous and next four-sentence lines, which effectively shrinks the training data lines to the $\frac{1}{4}$th of the original size. \citeA{voita-etal-2019-good}, who proposed the CADec, used a 6 million line EN$\rightarrow$RU parallel dataset that became 1.5 million in the second training step. In comparison, ours is almost 27x smaller, only 0.22 million in total and 0.055 million in the second step. An obvious way around this would be to use a sliding window of context sentences and generate up to three context sentences for each content sentence; however, we intended to maintain the original approach of the authors for these experiments.

For comparison, we also trained NMT models using WMT data (WMT data row in Table \ref{tab:mt-results}). These models reached 21.14 BLEU for {\enja} and 20.43 BLEU for {\jaen} on \textit{News commentary v15},\footnote{\url{http://www.statmt.org/wmt20/translation-task.html}} whose domain is close to the WMT data. However, based on our evaluation data, they underperformed our baselines trained on the proposed corpora. This shows that even with 60x training data, these models struggle to translate conversations. By combining all training data (bottom row in Table \ref{tab:mt-results}), the gain over the baselines is only 0.81--1.46 BLEU.


Figure \ref{fig:mt-comparison} shows one example of a Japanese sentence and its translations by the MT systems. There are no pronouns in the source sentence, but there is the noun 「方」, which should be translated into the English pronoun ``he,'' specifying the person to be the successor to the store. Both systems manage to translate this part correctly, but the baseline generates an additional pronoun in the end instead of ``the store.'' We observed many similar situations, where the contextual translation still did not match the reference and was not perfect, but the selection of pronouns had improved.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 6
\begin{figure}[t]
\begin{center}
\includegraphics{28-2ia3f6.pdf}
\end{center}
\hangcaption{{\jaen} translations of a sentence where the baseline generated an incorrect pronoun, but the concat. + factor system produced a more fitting translation.}
\label{fig:mt-comparison}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

To verify the usefulness of training on all three parts (AMI, BSD, ON) instead of just BSD, we performed a series of ablation experiments by training the baseline model configuration with different datasets. The two bottom rows of Table \ref{tab:mt-results} show that when training using only BSD the models perform considerably worse than with all three parts combined. However, without BSD in the training data the models performed very poorly on the BSD evaluation set, as expected. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Human Evaluation}
\label{sec:human-eval}

We translated the evaluation set in both directions using our baseline NMT and performed a two-step human evaluation similar to \citeA{voita-etal-2019-good}. Next, we analysed the remaining sentences to determine which sentences really require context.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Workflow}

We used Yahoo! Japan Crowdsourcing\footnote{\url{https://crowdsourcing.yahoo.co.jp/}} for human evaluation. We tried to guarantee the evaluation quality using screening questions that were indistinguishable from real questions. Only those who correctly answered all the screening questions were considered valid evaluators. Each sentence was evaluated by five evaluators.

In the first step, evaluators were asked to mark each sentence of the 2051 consecutive sentence pairs\footnote{The number of sentence pairs from each scenario is smaller than the number of sentences by one, because the first sentence of each scenario has no previous context. Therefore, the number of sentence pairs was $2{,}120 - 69 = 2{,}051$.} individually as OK or NG, where OK meant that the translation grasped the general meaning of the original sentence, whereas NG meant that the translation was completely unusable. In the second step, we only used sentence pairs where both sentences were marked OK in the first step by at least three evaluators. We then asked evaluators to mark each sentence pair as OK if the corresponding translations made sense in the context of each other or NG otherwise. We calculated the free-marginal kappa \shortcite{randolph2005free} values for the evaluations to measure the agreement between the evaluators. The results (overall agreement, 67\%; Free-marginal kappa, 0.34) showed moderate agreement, which is common for crowdsourcing.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 8
\begin{table}[b]
\input{03table08.tex}
\hangcaption{Results of the crowdsourcing human evaluation. The results of EN$\rightarrow$RU are only for reference from \protect\citeA{voita-etal-2019-good}.}
\label{tab:human-eval-table}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Analysis}

As a result of the crowdsourcing campaign (Table \ref{tab:human-eval-table}), we had 208 {\jaen} sentence pairs and 228 {\enja} sentence pairs marked as NG in the context of each other. Because we used the same method as \citeA{voita-etal-2019-good}, we also added their EN$\rightarrow$RU results to the table for reference. We employed two linguistic experts to check the translations along with their respective sources and references to determine their ambiguity and the need for additional context. In this step, they were also asked to categorise the ambiguity type. After the final step, 43 {\jaen} and nine {\enja} sentence pairs were marked as context-dependent. From the final results, we can conclude the following: 1) using the context may be beneficial for {\jaen} translations rather than {\enja} translations, and 2) the crowdsourcing evaluations are not reliable enough to draw direct conclusions even though the evaluation cost is low. From the former point, we can explain why the context information is beneficial for {\jaen} but not for {\enja} in Table \ref{tab:mt-results}. Context-aware models sometimes suffer from noise introduced from context sentences; in other words, the models can generate contents that are not related to the current source sentence but in the context sentences. The effect of noise induction might be larger than the benefit of using context for {\enja} translations.

Thirty-eight {\jaen} out of 43 pairs that were marked as context-dependent lack pronouns in the source sentence and did not have enough content to produce an unequivocal translation. One such example is shown in Figure \ref{fig:pronoun-example}. The MT system generated ``I'' as the subject of the source sentence, which is acceptable without any context. However, considering the previous sentence, using ``I'' as the subject becomes unnatural. Furthermore, if the speaker information is available (speaker information is available in our corpus) and the MT model can utilise that information, the subject should undoubtedly be ``you.'' The use of speaker information will be our future work.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 7
\begin{figure}[t]
\begin{center}
\includegraphics{28-2ia3f7.pdf}
\end{center}
\hangcaption{An example of {\jaen} MT output where there are no pronouns in the source and the MT system generates valid and fluent translations, but using the wrong pronouns -- \textit{I} instead of \textit{you}.}
\label{fig:pronoun-example}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The other five {\jaen} pairs contain ambiguous words or phrases that can be translated differently depending on the context. For example, 「1組」 can be translated as either ``one couple'' or ``one group.'' 

Similarly, in {\enja}, Chinese can refer to language (中国語) or food (中華料理), as shown in Figure \ref{fig:ambiguity-example}. Our best contextual models still struggle to translate such ambiguities, whereas slightly outperforming sentence-level baselines in handling pronouns. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 8
\begin{figure}[t]
\begin{center}
\includegraphics{28-2ia3f8.pdf}
\end{center}
\hangcaption{{\enja} MT output where {\it Chinese} is translated into ``中国語'' (Chinese language) instead of ``中華料理'' (Chinese food).}
\label{fig:ambiguity-example}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}
\label{sec:conclusion}

We presented a Japanese--English conversation parallel corpus intended for training and evaluation of MT systems. We describe the corpus in detail and indicate which linguistic phenomena are challenging for MT. In our evaluation set, we marked examples that can have multiple contrasting translations when tackled on the sentence-level. The release will include the full BSD corpus and Japanese translations of AMI and ON, along with instructions on how to align them. The original source language, speaker, scene, scenario/transcript, and ambiguity types will also be included. 

In the future, we plan to model speakers and origin languages in MT, as it can help capture the broader context \shortcite{maruf-etal-2018-contextual} of the conversations and more precise pronoun translations \shortcite{vanmassenhove-etal-2018-getting}. We also realise that BLEU is not sufficient for evaluating the translation of zero pronouns. To do this more thoroughly, we would need a dedicated dataset for zero-pronoun evaluation. We intend to use the dataset created by \shortciteA{shimazu-etal-2020-evaluation} for this task or create our own dataset on the proposed corpus. In addition, we also intend to conduct human evaluations focusing on pronoun translations.

It would also be interesting to conduct deeper analyses on the characteristics of the corpora, such as:
a) the effect of bidirectional translations on the BSD corpus; 
b) linguistic phenomena that exist only in the transcripts (AMI and ON); 
c) translationese effect at the Japanese side of AMI and ON.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment

This work was supported by the ``Research and Development of Deep Learning Technology for Advanced Multilingual Speech Translation'', the Commissioned Research of National Institute of Information and Communications Technology (NICT), JAPAN. We want to thank BAOBAB Inc.\footnote{\url{https://baobab-trees.com/}} for helping us construct the BSD corpus. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.6}
\bibliography{03refs}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\appendix

\section{Release Format Example}

The three sub-corpora are structured into and are released as json files. Scenarios/transcripts are gathered based on the corpora, and each scenario/transcript is constructed into a single json file, in which sentence pairs are represented as a list of json objects.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 9
\begin{figure}[b]
\begin{center}
\includegraphics{28-2ia3f9.pdf}
\end{center}
\caption{An example sentence pair from the Business Scene Dialogue sub-corpus. }
\label{fig:data-example}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Figure \ref{fig:data-example} shows a sentence pair taken from BSD as an example. Among all fields, ``no,'' ``speaker,''\footnote{For the BSD corpus, the speaker's names are taken from real names, but for the AMI and ON corpus, most of the names are anonymised into alphabets as they are not present in the original corpus.} ``en\_sentence,'' ``ja\_sentence,'' ``original\_language'' are common fields available in all the three parts of the corpus. Although most of the field names are self-explanatory, ``original\_language'' is noteworthy here. For AMI and ON, the field is set to English to indicate that both of the original corpora are in English. Conversely, in the case of BSD, it indicates in which language the monolingual scenarios are written in. Additionally, as BSD contains more information such as speaker's name in Japanese (ja\_speaker), scene of the scenario (tag), and title of the scenario (title), we include these fields in the final json files as well.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Human Evaluation Thresholds}

For both steps of the human evaluation process, we used a threshold of three concurrent evaluations to count a sentence or sentence pair as ``OK.'' Table \ref{tab:human-thresholds} shows how the evaluation results would change for higher and lower thresholds. In this table, the second step represents the case of using a threshold of three for the first step.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 9
\begin{table}[h]
\input{03table09.tex}
\caption{Human evaluation results for different thresholds. The one used (3) is marked in bold.}
\label{tab:human-thresholds}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\vspace{-1\Cvs}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Human Evaluation Agreement}

We calculated the free-marginal multirater kappa values for our human evaluations to measure the agreement between evaluators. The detailed agreement scores are listed in Table \ref{tab:human-agreement}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 10
\begin{table}[h]
\input{03table10.tex}
\caption{Human evaluation agreement scores for each evaluation step (1 and 2) and the average.}
\label{tab:human-agreement}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\vspace{-1\Cvs}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Mat\={\i}ss Rikters}{%
received his B.S., M.S. and Ph.D. degrees from the University of Latvia in 2012, 2014 and 2019 respectively. Currently, he is a postdoctoral researcher at the University of Tokyo. His main field of study is natural language processing and machine translation (MT) in particular. His work has mainly focused on the improvement of MT for low-resource and morphologically rich languages, and most recently - context-aware MT. 
}

\bioauthor[:]{Ryokan Ri}{%
is a first-year doctor's student at the Graduate School of Information Science and Technology, The University of Tokyo. His research interests include natural language processing, especially its multilingual applications.
}

\bioauthor[:]{Tong Li}{%
is currently a machine learning engineer at M3 Inc. He received his B.S. in Computer Science and Technology from Jilin University, M.S. in Information Science and Technology from the University of Tokyo in 2017 and 2020, respectively. 
}

\bioauthor[:]{Toshiaki Nakazawa}{%
received his B.S. in Information and Communication Engineering and M.S. in Information Science and Technology from the University of Tokyo in 2005 and 2007, respectively. He obtained his Ph.D. in Informatics from Kyoto University in 2010. He is currently a specially appointed lecturer at the University of Tokyo. His research interests center on natural language processing, particularly machine translation.
}

\end{biography}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\biodate






\end{document}

