\documentclass[english]{jnlp_3.5}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx,xcolor}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{hangcaption_jnlp}
\usepackage{alltt}
\usepackage{multirow}
\usepackage{array}
%%%%\usepackage{udline}
%%%%\setulminsep{1.2ex}{0.2ex}
%%%%\let\underline=\ul
\allowdisplaybreaks


\Volume{29}
\Number{2}
\Month{June}
\Year{2022}
%%%%\Issuetitle{Special Issue: XXXXXXXX}

\received{2021}{10}{19}
\revised{2022}{1}{30}
%%%%\rerevised{2006}{5}{6}
\accepted{2022}{3}{4}

\setcounter{page}{493}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Zero-Pronoun Annotation Support Tool for the Evaluation of Machine Translation on Conversational Texts}
\eauthor{Andre Rusli\affiref{Author_1} \and Makoto Shishido\affiref{Author_1}} 
\eabstract{%
This study proposes a support tool for building zero-pronoun evaluation sets called the zero-pronoun annotation support tool (0Past; pronounced zero-past). The proposed tool provides a chat-like user interface to facilitate the navigation of human annotators. Each conversation is displayed separately, and while the user views a certain conversation, the messages within the conversation are displayed individually with a distinct color for the newest message. Using 0Past, two zero-pronoun evaluation sets are constructed. These evaluation sets are then used to evaluate neural machine translation (NMT) models' performance translating Japanese conversations to English with the correct pronoun. Additionally, this study builds a zero-pronoun classification model by incorporating newly constructed evaluation sets and enables the tool to provide automated pre-annotation features, which can then be improved manually by human annotators. Finally, this study reports the evaluation results of training a Japanese-English neural machine translation model and compares its performance with two publicly available pretrained models in translating parallel conversational sentences from Japanese to English, which contains many omitted pronouns. The results confirm that phenomenon-specific evaluation sets are essential for better measuring NMT models when handling conversational sentences in Japanese, which is heavy on the anaphoric zero-pronoun phenomenon.
}
\ekeywords{Japanese-English Translation, Neural Machine Translation, Anaphoric Zero-pronoun, Support Tool, Evaluation Set}

\headauthor{Rusli and Shishido}
\headtitle{Zero-Pronoun Annotation Support Tool}

\affilabel{Author_1}{}{Tokyo Denki University}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%1 
\section{Introduction}

In recent years, many studies in the deep learning and natural language 
processing fields have progressed toward enabling the building of 
high-performance neural machine translation (NMT) models, most notably the 
neural encoder-decoder models with attention (Bahdanau et al. 2015; Vaswani 
et al. 2017). NMTs are a popular research field that is being applied in 
various forms of applied technologies to assist human communication. Many 
studies have achieved various translation milestones in various language 
directions, such as translation in low-resource languages (Hongzheng et al. 
2020; Mueller and Lal 2019; Yang et al. 2019). In some language directions 
and domains, sentence-level translation has been considered similar to human 
translation (Hassan et al. 2018). However, document-level translation 
remains a challenging field, particularly in the context of chat 
translation, in which sentences are typically spoken language instead of 
written language, and contain many references to previous sentences.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% Fig. 1
\begin{figure}[b]
\begin{center}
\includegraphics{29-2ia9f1.pdf}
\end{center}
\caption{An example of multilingual chat conversation using machine translation}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% Table 1
\begin{table}[b]
\input{09table01.tex}
\caption{Example of conversation containing sentences with omitted pronouns}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

As technology continues advancing, communicating with people in different 
languages via chat messages has become a trivial task. Various companies 
worldwide are already providing multilingual chat translation services and 
products (Unbabel 2021; Kotozna 2021; Language I/O 2021), as Fig.~1 
illustrates. A challenge in translating chat messages is the anaphoric 
zero-pronoun phenomenon, in which pronouns are dropped or omitted in a 
sentence. This arises from pronouns such as subjects, objects, and 
possessive cases are often omitted from conversational sentences. The 
sentences in Table 1 were taken from the business scene dialog corpus 
(Rikters et al. 2019), which contains parallel conversations in Japanese and 
English. This anaphoric zero-pronoun language is nonexclusive to Japanese. 
However, the surrounding words in a zero-pronoun Japanese sentence typically 
do not have inflectional forms that change depending on the previously 
omitted pronoun (Kudo et al. 2014), which makes the Japanese language one of 
the most difficult languages to resolve (Taira et al. 2021; Kudo et al. 
2014).

The main contribution of this study is to provide a support tool for 
building zero-pronoun evaluation sets, namely 0Past, to assist in the 
evaluation of NMT models translate heavy pro-drop languages, such as 
Japanese. This paper also reports two additional findings: (1) performance 
comparison, in terms of BLEU score, between three NMT models to translate 
chat messages from Japanese to English demonstrating that specific 
evaluation sets for zero-pronoun sentences are required; and (2) two 
zero-pronoun evaluation sets labeled using the proposed support tool.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%2 
\section{Related Works}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%2.1 
\subsection{NMT with Transformers}

Progress in NMT has been rapidly advancing in recent years. The machines' 
ability to translate certain language pairs at the sentence level, ignoring 
its contexts, is considered to perform competitively with human translation. 
Most state-of-the-art NMT models are based on the encoder-decoder 
architecture (Bahdanau et al. 2015) in which the encoder layers map the 
source sentence into word vectors, and the decoder layers produce the target 
sentence given the source sentences represented by its word vectors. This 
approach assumes a conditional independence of each sentence and then uses 
it to translate sentences from the source to the target language, ignoring 
both source- and target-side context sentences. As such, these systems 
optimize the negative log-likelihood of the sentences:
\begin{equation}
p(y^{(k)}│x^{(k)})= {\textstyle\prod^{n}_{t=1}} = (y_{t}^{(k)}│y_{<t}^{(k)},x^{(k)})
\end{equation}
where $x^{(k)}$ and $y^{(k)}$ are the k-th source and 
target training sentences, respectively, and $y_{t}^{(k)}$ is the t-th token in 
$y^{(k)}$ (Lopes et al. 2020).

Several studies have proposed the use of context-aware NMT approaches that 
integrate source- and target-side contexts (Tiedemann and Scherrer 2017; 
Zhang et al. 2018; Maruf et al. 2019; Voita et al. 2019). Existing studies 
have shown that human translators clearly outperform NMT models when the 
translation of the target language must consider the contexts at the 
document level in Chinese-English translations (Laubli et al. 2018; Toral et 
al. 2018). A previous study (Lopes et al. 2020) conducted a systematic 
comparison of context-aware NMT methods using large datasets that included 
findings for anaphoric pronoun translation. However, it focused only on 
three language directions: English to French, German, and Brazilian 
Portuguese. Studies related to the evaluation and improvement of discourse 
translation in Japanese, which has their own challenges, still lack 
comparisons to other languages.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%2.2 
\subsection{Corpora for Discourse Translation in Japanese}

Many of the publicly available parallel corpora for training a 
Japanese-English NMT model are constructed with sentences from written 
language, such as crawled documents from the web, patents (Goto et al. 
2011), and scientific papers (Nakazawa et al. 2016). Only in recent years 
has more work started to shift focus to providing corpora for discourse 
translation in Japanese. One research paper (Rikters et al. 2019) introduced 
a Japanese-English business conversation parallel corpus, the Business Scene 
Dialog (BSD) corpus. This corpus contains 955 scenarios with 30,000 parallel 
sentences. It provides a publicly available parallel corpus for training and 
evaluating conversational NMT for Japanese to English but lacks an 
evaluation set for measuring the models' capability to handle 
language-specific phenomena, such as anaphoric zero resolution.

Another study (Shimazu et al. 2020) separately constructed a test set for 
evaluating zero-pronoun resolution in Japanese-English translations, which 
consists of four components: a current sentence, context, a correct 
translation, and an incorrect translation. This type of test set is 
particularly useful because it contains zero pronoun information; therefore, 
translation models must predict the correct antecedent based on the contexts 
in previous sentences inside one conversation. However, manually 
handcrafting them requires time and effort; thus, such sets are rare, 
particularly those publicly available. Our research aims to kickstart the 
development of support tools that can handle specific language phenomena to 
assist the evaluation of model performance in discourse translation and, 
ultimately, improve the overall capability of machine translation models.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%2.3 
\subsection{Support Tool for Building Evaluation Sets}

Human involvement in various machine translation (MT) tasks, such as the 
creation and annotation of parallel corpora and manual evaluation, is 
considered important, although time-consuming and non-trivial. This is 
challenging for many reasons, one of which is that translation results 
produced by an NMT model are not guaranteed to be syntactically well-formed 
and easy to understand; thus, identifying the exact differences between 
translation candidates is challenging, particularly for a long or 
conversational document that contains many coreferences between sentences. 
Recently, the 2020 Conference on Machine Translation (WMT '20) developed a 
human intelligence task (HIT) tool (Farajian et al. 2020) based on a 
previously developed Appraise toolkit (Federmann 2010) to evaluate a shared 
tasks in chat translation. This tool provides a UI specifically designed to 
facilitate translation evaluation for the human evaluator in assessing the 
WMT '20 task submissions; however, although the tool provides modern UI for 
easier use, it focuses only on the human evaluation of machine translations, 
which are then compared to its machine counterparts to score the overall 
translation results without information regarding the model's capability in 
handling language-specific phenomena, such as anaphoric zero pronouns.

However, previous studies have proposed various tools to assist data 
annotation activities. WebAnno (Yimam et al. 2013) presented a 
general-purpose web-based annotation tool for a wide range of linguistic 
annotations, such as parts-of-speech called entities, dependency parsing, 
and co-reference chain annotations. In another study, INCEpTION (Klie et al. 
2018) provided an annotation platform for tasks, including interactive and 
semantic annotation. It is built based on the shortcomings of previous 
support tools, including WebAnno's split-screen mode, which was considered 
tedious for users. Such tools provide a wide range of features to support 
manual annotation, partially supported by pre-annotation features that 
leverage machine-learning methodologies. However, these tools were built for 
traditional NLP tasks tailored for document-like texts, such as 
parts-of-speech tagging and entity linking recommendations.

Our proposed tool, the Zero-Pronoun Annotation Support Tool (0Past), was 
designed and built to handle chat annotation and evaluation. For example, it 
outlines the boundaries between each speech of a certain speaker, 
facilitating navigation between conversations. The tool focuses on the 
anaphoric zero-pronoun phenomenon, which is a challenging task in chat 
translation on several languages, particularly in Japanese. The proposed 
tool is built on existing tools, tailored to support the annotation and 
evaluation of chat messages, to improve the performance of NMT models in 
handling sentences with anaphoric zero-pronouns. The following sections 
describe the complete list of current and potential features of 0Past.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%3 
\section{Proposed Tool: Zero Pronoun Annotation Support Tool (0Past)}

Training and evaluating machine translation are not straightforward 
processes. Many language-specific phenomena must be addressed when measuring 
the translation ability of an NMT model; one of which is the anaphoric 
zero-pronoun, which is especially challenging in Japanese conversational 
sentences. To build evaluation sets for the zero-pronoun phenomenon in MT, 
parallel datasets must be further annotated with information on whether a 
Japanese sentence contains an anaphoric zero-pronoun. Although this manual 
process is important, it is time consuming. This study proposes a support 
tool, the Zero-Pronoun Annotation Support Tool (0Past, pronounced 
zero-past), to assist in the creation of zero-pronoun-annotated parallel 
datasets. The annotated dataset can then be used as an evaluation set to 
measure NMT model performance. This research used two primary parallel 
datasets when building the support tool, the Business Scene Dialogue (BSD) 
and Travel Conversation Exercise (TCE) corpora, which are further explained 
in subsequent sections.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% Fig. 2
\begin{figure}[b]
\begin{center}
\includegraphics{29-2ia9f2.pdf}
\end{center}
\caption{Use-case diagram of the proposed tool, 0Past}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%3.1 
\subsection{Features and User Interface of 0Past}

The proposed tool was developed using VueJS\footnote{\url{http://vuejs.org}} and Buefy\footnote{\url{http://buefy.org}} as the 
front-end frameworks and the Firebase Realtime Database\footnote{\url{https://firebase.google.com/docs/database}} to store 
annotations. Fig.~2 shows a use-case diagram of the proposed tool with two 
types of users: a regular user (User) and an administrator (Admin). Regular 
users can perform three actions: upload a new parallel corpus, annotating an 
uploaded corpus, and download the annotation data. In turn, the 
administrator can perform two actions: manage the uploaded corpus inside the 
application and validate annotations made by the users, which can later be 
used to create the aggregated annotation of a certain dataset. Fig.~3 shows 
the About page, which explains the anaphoric zero-pronoun with three 
navigation items: Home, Upload, and About. Home is the page where users can 
select a corpus to annotate and download its annotation data, whereas Upload 
provides the interface to upload new corpora. The following paragraphs 
explain these features in detail.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% Fig. 3
\begin{figure}[b]
\begin{center}
\includegraphics{29-2ia9f3.pdf}
\end{center}
\caption{About anaphoric zero-pronoun in 0Past}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% Table 2
\begin{table}[b]
\input{09table02.tex}
\caption{Content format inside a corpus file to be uploaded to 0Past}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The first feature allows users to upload parallel corpora that are currently 
unavailable in the system. A specific format is available for download that 
the corpus must follow. Thus, users must follow the correct format, and the 
system checks this before enabling the upload. The format used in the 
current version of 0Past follows the JSON format, as shown in Table 2. By 
default, the uploaded corpus can only be accessed by the uploader. However, 
users can also choose whether the uploaded corpus can be accessed publicly. 
If a corpus is set as public, then other users of 0Past can access it and 
provide their annotations, which are saved in each of the user's account.

Next is the primary feature of 0Past, which is users viewing uploaded 
parallel corpora and contributing to the community by providing their 
zero-pronoun annotations for the available corpora. Fig.~4 depicts the main 
user interface for annotation, where users can navigate between sentences 
and conversations within a dataset. The proposed tool provides a chat-like 
user interface and displays each conversation separately. While the user 
views a certain conversation, the messages within the conversation are 
displayed independently with a distinct color for the newest message. Users 
can see messages back and forth in the conversation with the speaker's name 
displayed above along with the chat number, allowing them to have a sense of 
direction in terms of where they are in the conversation. Users can then 
annotate each sentence by selecting whether the zero-pronoun phenomenon 
exists in the corresponding sentence.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% Fig. 4
\begin{figure}[b]
\begin{center}
\includegraphics{29-2ia9f4.pdf}
\end{center}
\caption{Sentences and their annotations from the TCE corpus displayed in 0Past}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Furthermore, in addition to labeling each sentence as a zero-pronoun or not, 
users can also annotate which of the previous sentences, if any, act as 
context sentences that decide the content of the omitted pronoun. As shown 
in Fig.~5, the current sentence is ``そうですね、現在の販売高は 400 
万ドル前後です。'' and is labeled a zero-pronoun sentence. In English, this 
sentence is ``Well, we are doing around four million in sales right now.'' 
The omitted pronoun is ``we,'' however, knowing this by looking at this 
sentence only is difficult, even for humans. However, if we look at the 
previous sentences, some sentences provide contexts, thereby helping us 
decide that the omitted pronoun is indeed ``we.'' To provide this type of 
annotation, users can scroll to see the previous sentences and click on the 
previous sentences that provide the context. The clicked sentence is then 
colored green, as shown in Fig.~5.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% Fig. 5
\begin{figure}[b]
\vspace{-0.5\Cvs}
\begin{center}
\includegraphics{29-2ia9f5.pdf}
\end{center}
\caption{Annotating a previous sentence with context related to the current zero-pronoun sentence}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% Fig. 6
\begin{figure}[b]
\begin{center}
\includegraphics{29-2ia9f6.pdf}
\end{center}
\caption{Options for downloading annotated data}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The last feature enables users to download a parallel corpus that is already 
annotated with zero-pronoun related labels, which can be either a corpus 
that has been labeled individually by the user or a corpus with aggregated 
annotations labeled by multiple 0Past users. For the corpus individually 
labeled by the user, the system checks and warns the user if unlabeled data 
can still be downloaded if the user intends to. For the corpus labeled by 
multiple 0Past users, the tool merges annotations from users that have been 
confirmed and aggregated manually by the administrators and allows the user 
to download the final aggregated corpus. In the current version, the 
administrators of 0Past are the authors of this paper; hence, the user is 
unable to download the aggregation annotation data before it is accepted and 
enabled by administrators. Fig.~6 shows the two buttons to download the 
annotate; note that the aggregated annotation button is disabled when 
available data has not been provided by the administrators. There are 
several methods to use annotations from multiple users to derive a single 
ground-truth dataset, such as majority voting and averaging. We aim to 
address this issue and implement such methods for improving the 
crowdsourcing feature in the next version of 0Past; for example, by adding 
an automated aggregation function.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%3.2 
\subsection{Zero-Pronoun Evaluation Sets}

After developing the primary features of 0Past, we annotated each sentence 
from the BSD and TCE corpora with a zero-pronoun label. We performed this 
annotation process by navigating each sentence in each conversation and 
deciding whether a sentence written in Japanese contained an omitted 
pronoun. If it did, the sentence was labeled ``Zero-Pronoun'' as a 
zero-pronoun sentence, ``Nope'' otherwise. Fig.~4 presents the user 
interface. Furthermore, if it is a zero-pronoun sentence, we look at the 
previous sentences in the same conversation and select the sentence(s) that 
provide context for the omitted pronoun. After the annotation processes were 
completed, we downloaded the annotations to be used as evaluation sets. The 
downloaded data are in JSON format, similar to the structure shown in Table 
2, with two additional key fields: \texttt{zeroPronoun} (Boolean) and 
\texttt{contextSentences} (Array). The \texttt{zeroPronoun} field contains information on 
whether a sentence is a zero-pronoun (true) or not (false). The 
\texttt{contextSentence} field is an array of numbers/integers containing the 
sentence number of the previous sentence(s) selected as context(s). Table 3 
shows an example of zero-pronoun annotation on the downloaded evaluation set 
based on the BSD corpus development set.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% Table 3
\begin{table}[t]
\input{09table03.tex}
\caption{Example of zero-pronoun annotation on a chat sentence}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%4 
\section{Experiments and Results}

This section describes the structure and results of the experiments. In 
Section 4.1, we describe the datasets used in the various stages of our 
experiments. For annotation and evaluation purposes, this study used two 
parallel corpora containing spoken conversational sentences. The first 
evaluation corpus was taken from the development set of the Business Scene 
Dialogue (BSD) corpus (Rikters et al. 2019), which contains conversations 
occurring in the business domain. The second corpus is a collection of 
parallel sentences that we assembled, the Travel Conversation Exercise (TCE) 
corpus, which contains example conversations that typically occur when 
traveling to foreign countries.

Section 4.2 reports how we annotated two sets of Japanese-English parallel 
corpora using 0Past. We created two evaluation sets to assess the MT model 
when translating sentences that contain anaphoric zero-pronouns, 
particularly from Japanese to English. Additionally, using the annotated 
evaluation set, we trained a model to classify the zero-pronoun label, given 
a pair of Japanese and English sentences, by fine-tuning the XLM-RoBERTa 
sequence classification model. This shows the potential for an improvement 
to 0Past, enabling the tool to provide pre-annotations that can assist 
annotators, particularly when handling a large number of sentences.

In Section 4.3, we report our experiments on how we can leverage such 
zero-pronoun evaluation sets to assess the performance of MT models in a 
conversational setting. This study uses three NMT models to translate the 
above evaluation sets, that is, the development set of the BSD and TCE 
corpora. This experiment aims to compare the performance of existing NMT 
models when translating conversational sentences with and without 
zero-pronouns from Japanese to English. The results confirmed that 
zero-pronoun evaluation sets are essential for supporting the automated 
evaluation of NMT models.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%4.1 
\subsection{Data Preparation}

For annotation and evaluation purposes, this study used two parallel corpora 
containing conversational sentences in Japanese and English, which is 
particularly challenging in terms of the anaphoric zero-pronoun. The first 
corpus was developed from the Business Scene Dialogue (BSD) corpus. This 
development set contained 69 business conversations, with 2,051 parallel 
sentences. The second corpus was a Japanese-English parallel corpus 
assembled from our laboratory's archive, based on a previous study (Shishido 
1998). The assembled conversations were also used officially to teach 
English as a foreign language to Japanese university students. We refer to 
this as the Travel Conversation Exercise (TCE) corpus. It contained various 
sentences for teaching English as a second language to university students 
in Japan. There were 888 parallel sentences written in Japanese and English, 
all of which were sample conversations that may occur when a student travels 
abroad. Some of the conversation themes included (but not limited to) 
airports, hotel reservations, sightseeing, and emergencies. Owing to the 
limited number of sentences, we only used this corpus as an evaluation set 
and did not include them for training. Both corpora follow a similar format: 
a JSON array containing many conversations (written in Japanese and English) 
by two or more speakers. For each conversation, an array contains sentences 
in the same format, as shown in Table 2.

Additionally, to build the NMT model that used for comparison in Section 
4.3, we used the training set of the BSD corpus to train our NMT model; 
however, as the number of sentences was limited (approximately 20,000 
sentences), two additional parallel datasets from other sources were also 
used to pretrain the model. The first was the Japanese version of the 
ParaCrawl dataset and JParaCrawl corpus (Morishita et al. 2020). It is 
claimed to be the largest publicly available English-Japanese parallel 
corpus created by NTT Communication Science Laboratories.\footnote{\url{http://www.kecl.ntt.co.jp/icl/lirg/jparacrawl/}} It contains 
approximately 10 million parallel sentences and was created by web crawling 
and automatically aligning parallel sentences. JParaCrawl has been shown to 
include a wider range of domains than previously available English-Japanese 
parallel datasets and suitable for training a neural translation model that 
works well as a pretrained model that can then be fine-tuned to specific 
domains. The second dataset was the Japanese English Subtitle Corpus (JESC) 
(Pryzant et al. 2018), which contains approximately 2.8 million sentences 
consisting of sentences from movies and TV subtitles. Unlike JParaCrawl, 
JESC contains a conversational dialog extracted from movie subtitles and is 
claimed to be the largest freely available dataset of its kind. JESC\footnote{\url{https://nlp.stanford.edu/projects/jesc/}} 
was built by crawling subtitles from four free and open subtitle 
repositories, performing several preprocessing steps to convert them into a 
form suitable for alignment, and finally combining them into one dataset 
consisting of 29,368 unique English words and 87,833 unique Japanese words. 
All three datasets used in this study are publicly available for download 
from their websites.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%4.2 
\subsection{Zero Pronoun Sentence Classification for Pre-Annotation}

Using the evaluation sets that were previously labeled using 0Past, we 
conducted an additional experiment to explore the possibility of building a 
classification model to predict whether a pair of Japanese-English sentences 
contains an anaphoric zero-pronoun. This experiment aims to determine if we 
can use the model to provide an automated pre-annotation feature for a new 
parallel corpus, in which predicted labels can be updated manually by human 
annotators. The model was trained by fine-tuning the pretrained XLM-RoBERTa 
(Conneau et al. 2020) base model with a vocabulary size of 250,002 and a 
maximum length of 512. This experiment used XLM-R primarily because of its 
capability to handle cross-lingual sentences, public availability, and ease 
of use. First, we combined the two annotated evaluation sets, the BSD and 
TCE corpora, to build a collection of 2,939 pairs of parallel sentences, 
each with its zero-pronoun label. There were 1,762 pairs of sentences marked 
as zero-pronouns and 1,177 pairs of sentences marked as non-zero-pronouns. 
The data were then split for training with a train-validation ratio of 
80:20, and the model was trained for four epochs with a batch size of 32. 
AdamW was used as the optimizer, with a learning rate of $2.0 \times 10^{-5}$ and epsilon of 
$1.0 \times 10^{-8}$. SentencePiece was also used by XLM-R to tokenize both Japanese and 
English sentences, eliminating the need to prepare a specialized tokenizer 
for each language.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% Table 4
\begin{table}[b]
\input{09table04.tex}
\hangcaption{Zero-pronoun label prediction of sampled sentences from the test set ($1 = \mbox{zero-pronoun}$, $0 =$ not zero-pronoun)}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% Table 5
\begin{table}[b]
\input{09table05.tex}
\hangcaption{Zero-pronoun label prediction of newly created sentences ($1 = \text{zero-pronoun}$, $0 =$ not zero-pronoun)}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The training phase completed with an average of 32 seconds per epoch, and 
the trained model performed with 0.88 accuracy on the validation set, 
meaning that, given a pair of sentences in Japanese and English, the model 
can correctly predict whether most sentences contain an omitted pronoun. 
Table 4 shows some of the model's predictions on the sampled pairs of 
sentences in the test set, and Table 5 shows its predictions on new sentence 
pairs (those that do not exist in either the BSD or TCE corpora). The 
combined dataset contained 1,762 zero-pronoun and 1,117 non-zero-pronoun 
sentences. This experiment, although early, shows another potential benefit 
by enabling the tool to provide automated pre-annotations on a newly 
uploaded corpus, reducing the efforts required for manual work but with 
satisfying results.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%4.3 
\subsection{Neural Machine Translation Experiments}

We conduct experiments to show how to leverage such zero-pronoun evaluation 
sets to assess the machine translation model performance in a conversational 
setting. First, we describe the training process for building an NMT to 
translate conversational sentences from Japanese to English. Our model is 
based on sequence-to-sequence transformers (Vaswani et al. 2017). 
Additionally, for comparison purposes and better reproducibility, we used 
two publicly available pretrained Japanese-English NMT models: the Opus-MT 
Japanese-English model\footnote{\url{https://huggingface.co/Helsinki-NLP/opus-mt-ja-en}} (Tiedemann and Thottingal 2020), and the M2M100 
model\footnote{\url{https://huggingface.co/facebook/m2m100_418M}} (Fan et al. 2021). Unlike our model, these models have been 
extensively evaluated and trained using a large amount of data and more 
complex architectures. Finally, this paper reports the translation 
performance of the three models for conversational sentences, which were 
previously annotated using 0Past, with and without zero-pronouns.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%4.3.1 
\subsubsection{Training Configuration and Specifications}

All experiments reported in this paper were conducted using a PC with a 
single NVIDIA GeForce RTX 3070 GPU and an Intel(R) Core (TM) i9-10900 CPU 
with 32~GB of RAM. Owing to limited resources, a considerable gap in 
performance may exist, measured in the BLEU score, between our own model and 
state-of-the-art NMT models. This matches our objectives, which do not 
include building a state-of-the-art translation model, emphasizing the need 
for phenomenon-specific evaluation for MT and confirming the need for a 
support tool for building evaluation sets to better support the training and 
evaluation of future NMT models, focusing on those capable of better 
anaphoric zero pronoun resolution from Japanese to English.

Our model was based on a transformer model (Vaswani et al. 2017). After 
experimenting with several hyperparameter combinations (Bawden et al. 2018; 
Hieber et al. 2017), we adjusted parameters to fit these limitations. 
Initially, we trained the models with 16 epochs; however, we gradually 
reduced the epochs after observing that the model performance did not 
significantly improve on the development set. The final models were trained with 8 
epochs using an encoder/decoder with three and six layers. The model uses an 
embedding size and feedforward embedding size of 512 with 8 attention heads. 
For the optimizer, the model was trained using Adam, learning rate was set 
to 0.0001, and the betas were 0.9 and 0.98, and epsilon was $-1.0\times 10^{-9}$. 
SentencePiece (Kudo and Richardson 2018) was used to tokenize both English and 
Japanese sentences with a vocabulary size of 32,000.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%4.3.2 
\subsubsection{Performance Evaluation: Overall BLEU Score}

After training our model, we measured its performance on the BSD and TCE 
datasets using BLEU scores (Papineni et al. 2002). The experiments trained 
several models using three and six encoder-decoder layers and tried several 
combinations of training datasets, including JParaCrawl only, 
JParaCrawl$+$JESC, and JParaCrawl$+$JESC$+$BSD corpora training sets along 
with their backtranslations. Table 6 shows that adding the BSD training set 
containing approximately 20,000 sentences to the two larger corpora, 
JParaCrawl and JESC, significantly improved performance. The best 
performance on the BSD corpus development set was achieved by a combination 
of three encoder-decoder layers with the JParaCrawl, JESC, and BSD corpus 
training set and its backtranslations with a BLEU score of 10.2. In summary, 
we call this model ``our final model'' in subsequent sections and 
subsections.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% Table 6
\begin{table}[b]
\input{09table06.tex}
\caption{NMT model training performance on BSD corpus development set}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We then compared the model's performance with the M2M100 and Opus-MT 
Japanese to English model to translate the BSD and TCE corpora. Table 7 
compares the BLEU scores, which are the averaged results from several runs 
with different random seeds. Evidently, for both datasets, the BSD and TCE 
corpora, the Opus-MT model for the Japanese-English translation direction 
achieved the best BLEU score. This is possibly because, compared to our 
final model, the Opus-MT model was trained using a larger dataset and 
architecture; thus, the BLEU score is better, but the inference time is 
longer. Furthermore, M2M100 is a huge cross-lingual language that can 
translate various language directions. In contrast, the Opus-MT model we 
used was trained specifically for Japanese to English, thereby dropping the 
BLEU score for M2M100 in this direction and increasing the inference time 
owing to the larger size of the model.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% Table 7
\begin{table}[b]
\input{09table07.tex}
\hangcaption{Performance comparison between our final model, Opus-MT and M2M100 on BSD development set and TCE corpora}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% Table 8
\begin{table}[b]
\input{09table08.tex}
\caption{Sentences with and without zero-pronoun translated using our model, Opus-MT, and M2M100}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%4.3.3 
\subsubsection{Performance Evaluation: Zero-Pronoun Translation}

In addition to measuring the BLEU score and time elapsed during inference, 
we manually examined some of the translated sentences using the three 
models. The first sentence in Table 8 is an example sentence without omitted 
pronouns. All three models can correctly predict the pronoun, as stated 
clearly in the original sentence; however, the Opus-MT model, which is 
trained specifically with Japanese-English data and a larger architecture 
than our model, provided the best result. This matches the BLEU score 
results from the previous subsection, which confirm that Opus-MT performs 
better than the other models. However, in the second example, where several 
pronouns are omitted, our model could translate and predict the translation 
well, particularly when compared with the M2M100 model. Furthermore, while 
both sentences provided by Opus-MT and our model translates correctly, the 
Opus-MT added a pronoun ``you'' that does not exist in the reference.

The results in Table 8 also support the claims made by previous studies 
(Laubli et al. 2018; Toral et al. 2018; Hardmeier 2012), in that merely 
comparing words and characters in translated sentences and their references 
is insufficient to evaluate MT performance in discourse. An NMT model may 
perform better in translating an entire sentence to mispredict or even 
completely miss the omitted pronoun in the original sentence. This also 
shows that training an NMT model by looking at isolated sentences 
independently without context could result in poor performance for 
conversational sentences, even for huge transformer-based models.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% Table 9
\begin{table}[b]
\input{09table09.tex}
\hangcaption{Gaps in BLEU score when translating zero-pronoun sentences in the BSD corpus development set}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

As explained in Section 3, this research annotates each sentence in the BSD 
(development set) and TCE corpora with a zero-pronoun label using 0Past. The 
BSD corpus contains 1,195 sentences annotated with the zero-pronoun label, 
whereas the TCE corpus contains 567 sentences annotated with the 
zero-pronoun label. We separated the translation results of sentences with 
and without zero-pronouns and calculated the BLEU scores of the three NMT 
models from the previous sections. Table 9 shows the gap in the BLEU scores 
achieved by each model when translating sentences with and without the 
zero-pronoun phenomenon from the BSD corpus. Although M2M100 has the largest 
gap, 2.4 points in the BLEU score, a similar trend is visible in the two 
other models, which shows that the score is smaller for zero-pronoun 
sentences. These results confirm that specialized evaluation sets are 
necessary to better measure NMT models' performance for language-specific 
phenomena, particularly anaphoric zero-pronouns in Japanese, which is the 
focus of this research. The proposed tool is expected to support researchers 
and practitioners in building evaluation sets with better quality and 
quantity, with less time and effort.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%5 
\section{Conclusion}

This study proposes a zero-pronoun annotation support tool called 0Past. The 
primary feature of the current 0Past is the annotation of parallel sentences 
with zero-pronoun information, given a set of sentences displayed in 
Japanese and English. To accomplish this, 0Past shows the sentences in a 
chat-like interface, enabling users to see past utterances inside a 
conversation to understand the context of the sentence being annotated. 
Additionally, this paper proposes two new zero-pronoun evaluation sets: (1) 
a Japanese-English parallel corpus called the Travel Conversation Exercise 
(TCE) corpus, which contains 888 dialog sentences and their zero-pronoun 
annotations and (2) a zero-pronoun-annotated dataset based on the BSD 
corpus. This study also built a binary classification model with the labeled 
BSD and TCE corpora to predict whether a sentence contains an omitted 
pronoun using the XLM-R cross-lingual model. Early results showed that the 
model achieves an 88{\%} accuracy on the test set and an improvement that 
may provide automated pre-annotations, which can then be updated by human 
annotators. Furthermore, this paper compares three NMT models in translating 
conversational Japanese sentences into English that contain many omitted 
pronouns. The results confirm that these evaluation sets are essential to 
better measure the performance of NMT models for language-specific 
phenomena, particularly anaphoric zero-pronouns in Japanese. The proposed 
tool is expected to support researchers and practitioners in building 
evaluation sets with better quality and quantity, with less time and effort.

Future research steps include incorporating more data with more information 
and better quality, such as gathering more data and adding better 
annotations using crowdsourcing approaches and professional translators. We 
plan to continue improving this tool by open-sourcing and adding more 
features to support the creation of evaluation sets, such as providing a 
reliable automated aggregation system for crowdsourcing annotations and 
adding gamification elements to encourage people to contribute to the 
creation of open datasets. Finally, it is also worth exploring various 
evaluation metrics that suit the need of MT models for conversational 
sentences, particularly to measure anaphoric zero-pronoun resolution 
performance. For instance, a study has proposed the use of antecedent 
F-measure (Taira et al. 2021). The antecedent F-measure has been reported to 
capture models' capability to solve the zero-pronoun problem in a more 
suitable manner, as the effectiveness of zero-pronoun resolution differs 
depending on the type and case of each zero pronoun.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment

This work was partially supported by the Research Institute for Science and 
Technology of Tokyo Denki University Grant Number Q21D-01/Japan.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.6}
\begin{thebibliography}{}

\item
Bahdanau, D., Cho, K., and Bengio, Y. (2015). ``Neural Machine Translation 
by Jointly Learning to Align and Translate.'' \textit{arXiv preprint arXiv:1409.0473}.

\item
Bawden, R., Sennrich, R., Birch, A., and Haddow, B. (2018). ``Evaluating 
Discourse Phenomena in Neural Machine Translation.'' In \textit{Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies}, Volume 1 (Long 
Papers), pp. 1304--1313. New Orleans.

\item
Conneau, A., Khandelwal, K., Goyal, N., Chaudhary, V., Wenzek, G., 
Guzm\'{a}n, F., Stoyanov, V. (2020). ``Unsupervised Cross-lingual 
Representation Learning at Scale.'' In \textit{The 58th Annual Meeting of the Association for Computational Linguistics}, pp. 8440--8451. Online.

\item
Fan, A., Bhosale, S., Schwenk, H., Ma, Z., El-Kishky, A., Goyal, S., Baines, 
M., Celebi, O., Wenzek, G., Chaudhary, V., Goyal, N., Birch, T., 
Liptchinsky, V., Edunov, S., Grave, E., Auli, M., and Joulin, A. (2021). 
``Beyond English-Centric Multilingual Machine Translation.'' \textit{Journal of Machine Learning Research}, \textbf{22}, 
pp.~1--48.

\item
Farajian, M. A., Lopes, A. V., Martins, A. F., Maruf, S., and Haffari, G. 
(2020). ``Findings of the WMT 2020 Shared Task on Chat Translation.'' In 
\textit{5th Conference on Machine Translation (WMT)}, pp. 65--75. Online.

\item
Federmann, C. (2010). ``Appraise: An Open-Source Toolkit for Manual 
Phrase-Based Evaluation of Translations.'' In \textit{the 7th International Conference on Language Resources and Evaluation}, pp. 25--35. Valletta, Malta.

\item
Goto, I., Lu, B., Chow, K. P., Sumita, E., and Tsou, B. K. (2011). 
``Overview of the Patent Machine Translation Task at the NTCIR-9 Workshop.'' 
In \textit{NTCIR-9 Workshop Meeting}, pp.~559--578. Tokyo, Japan.

\item
Hardmeier, C. (2012). ``Discourse in Statistical Machine Translation: A 
Survey and a Case Study.'' \textit{Discours, Discourse in Statistical Machine Translation}, 11.

\item
Hassan, H., Aue, A., Chen, C., Chowdhary, V., Clark, J., Federmann, C., Wu, 
L. (2018). ``Achieving Human Parity on Automatic Chinese to English News 
Translation.'' \textit{arXiv preprint arXiv:1803.05567}.

\item
Hieber, F., Domhan, T., Denkowski, M., Vilar, D., Sokolov, A., Clifton, A., 
and Post, M. (2017). ``Sockeye: A Toolkit for Neural Machine Translation.'' 
\textit{arXiv preprint arXiv:1712.05690}.

\item
Hongzheng, L., Jiu, S., and Can, S. (2020). ``Revisiting Back-Translation 
for Low-Resource Machine Translation Between Chinese and Vietnamese.'' \textit{IEEE Access}, 
\textbf{8}(1), 119931--119939.

\item
Klie, J., Bugert, M., Boullosa, B., de Castilho, R. E., Gurevych, I. (2018). 
``The INCEpTION Platform: Machine-Assisted and Knowledge-Oriented 
Interactive Annotation.'' In \textit{Proceedings of the 27th International Conference on Computational Linguistics: System Demonstrations}, pp. 5--9. Santa Fe, New Mexico, USA.

\item
Kotozna. (2021). Kotozna. Accessed on August 28, 2021, from 
\url{https://kotozna.com/}.

\item
Kudo, T., and Richardson, J. (2018). ``SentencePiece: A Simple and Language 
Independent Subword Tokenizer and Detokenizer for Neural Text Processing.'' 
In \textit{Conference on Empirical Methods in Natural Language Processing}, pp. 66--71. Brussels, Belgium.

\item
Kudo, T., Ichikawa, H., and Kazawa, H. (2014). ``A Joint Inference of Deep 
Case Analysis and Zero Subject Generation for Japanese-to-English 
Statistical Machine Translation.'' In \textit{Annual Meeting of the Association for Computational Linguistics (Short Papers)}, pp. 557--562. Baltimore.

\item
Language I/O. (2021). Language I/O. Accessed on August 28, 2021, from 
\url{https://languageio.com/}.

\item
Laubli, S., Sennrich, R., and Volk, M. (2018). ``Has Machine Translation 
Achieved Human Parity? A Case for Document-level Evaluation.'' In 
\textit{Conference on Empirical Methods in Natural Language Processing}, pp. 4791--4796. Brussels, Belgium.

\item
Lopes, A., Farajian, M. A., Bawden, R., Zhang, M., and Martins, A. F. 
(2020). ``Document-level Neural MT: A Systematic Comparison.'' In \textit{22nd Annual Conference of the European Association for Machine Translation}, pp. 225--234. Lisboa, 
Portugal.

\item
Maruf, S., Martins, A. F., and Haffari, G. (2019). ``Selective Attention for 
Context-aware Neural Machine Translation.'' In \textit{Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies}, Volume 1 (Long and Short Papers), pp. 3092--3102. Minneapolis.

\item
Morishita, M., Suzuki, J., and Nagata, M. (2020). ``JParaCrawl: A Large 
Scale Web-Based English-Japanese Parallel Corpus.'' In \textit{the 12th Language Resources and Evaluation Conference}, pp. 3603--3609. Marseille, France.

\item
Mueller, A., and Lal, Y. K. (2019). ``Sentence-Level Adaptation for 
Low-Resource Neural Machine Translation.'' In \textit{Workshop on Technologies for MT of Low Resource Languages}, pp. 39--47. Dublin, Ireland.

\item
Nakazawa, T., Yaguchi, M., Uchimoto, K., Utiyama, M., Sumita, E., Kurohashi, 
S., and Isahara, H. (2016). ``ASPEC: Asian Scientific Paper Excerpt 
Corpus.'' In \textit{10th International Conference on Language Resources and Evaluation (LREC'16)}, pp. 2204--2208. Portoro\v{z}, Slovenia.

\item
Papineni, K., Roukos, S., Ward, T., and Zhu, W.-J. (2002). ``BLEU: A Method 
for Automatic Evaluation of Machine Translation.'' In \textit{40th Annual Meeting of the Association for Computational Linguistics (ACL)}, pp. 311--318. Philadelphia.

\item
Pryzant, R., Chung, Y., Jurafsky, D., and Britz, D. (2018). ``JESC: 
Japanese-English Subtitle Corpus.'' In \textit{the 11th International Conference on Language Resources and Evaluation (LREC 2018)}, pp. 1133--1137. Miyazaki, Japan.

\item
Rikters, M., Ri, R., Li, T., and Nakazawa, T. (2019). ``Designing the 
Business Conversation Corpus.'' In \textit{Workshop on Asian Translation}, pp. 54--61. Hongkong.

\item
Shimazu, S., Takase, S., Nakazawa, T., and Okazaki, N. (2020). ``Evaluation 
Dataset for Zero Pronoun in Japanese to English Translation.'' In 
\textit{Conference on Language Resources and Evaluation (LREC 2020)}, pp. 3630--3634. Marseille.

\item
Shishido, M., (1998). 逆襲の英会話海外旅行編．旺文社．[Shishido, M. (1998). 
Gyakush\={u} no eikaiwa kaigairyok\={o}-hen. \={O}bunsha.]

\item
Taira, H., Sudoh, K., and Nagata, M. (2021). ``Zero Pronoun Resolution can 
Improve the Quality of J-E Translation.'' In \textit{6th Workshop on Syntax, Semantics and Structure in Statistical Translation}, pp. 111--118. Jeju, South Korea.

\item
Tiedemann, J., and Scherrer, Y. (2017). ``Neural Machine Translation with 
Extended Context.'' In \textit{3rd Workshop on Discourse in Machine Translation}, pp. 82--92. Copenhagen, Denmark.

\item
Tiedemann, J., and Thottingal, S. (2020). ``OPUS-MT -- Building Open 
Translation Services for the World.'' In \textit{The 22nd Annual Conference of the European Association for Machine Translation}, pp. 479--480. Lisboa, Portugal: European 
Association for Machine Translation.

\item
Toral, A., Castilho, S., Hu, K., and Way, A. (2018). ``Attaining the 
Unattainable? Reassessing Claims of Human Parity in Neural Machine 
Translation.'' In \textit{Proceedings of the 3rd Conference on Machine Translation: Research Papers}, pp. 113--123. Brussels, Belgium.

\item
Unbabel. (2021). Unbabel. Retrieved August 28, 2021, from 
\url{https://unbabel.com}

\item
Vaswani, A., Shazeer, N., Parmar, N., Uszkoreit, J., Jones, L., Gomez, A. 
N., Kaiser, {\L}., and Polosukhin, I. (2017). ``Attention Is All You Need.'' In 
\textit{31st Conference on Neural Information Processing Systems}, pp. 5998--6008. Long Beach, CA.

\item
Voita, E., Sennrich, R., and Titov, I. (2019). ``When a Good Translation is 
Wrong in Context: Context-Aware Machine Translation Improves on Deixis, 
Ellipsis, and Lexical Cohesion.'' In \textit{57th Annual Meeting of the Association for Computational Linguistics}, pp. 1198--1212. Florence, Italy.

\item
Yang, M., Wang, R., Chen, K., Utiyama, M., Sumita, E., Zhang, M., and Zhao, 
T. (2019). ``Sentence-Level Agreement for Neural Machine Translation.'' In 
\textit{Annual Meeting of the Association for Computational Linguistics}, pp. 3076--3082. Florence, Italy.

\item
Yimam, S.M., Gurevych, I., de Castilho, R. E., Biemann, C. (2013). 
``WebAnno: A Flexible, Web-based and Visually Supported System for 
Distributed Annotations.'' In \textit{Proceedings of the 51st Annual Meeting of the Association for Computational Linguistics}, pp. 1--6. Sofia, Bulgaria.

\item
Zhang, J., Luan, H., Sun, M., Zhai, F., Xu, J., Zhang, M., and Liu, Y. 
(2018). ``Improving the Transformer Translation Model with Document-Level 
Context.'' In \textit{Conference on Empirical Methods in Natural Language Processing}, pp. 533--542. Brussels.

\end{thebibliography}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Andre Rusli}{%
is currently pursuing a Ph.D. in Engineering at Tokyo 
Denki University, Japan. He received his master's degree from the same 
university in 2017. His research interests include cross-lingual natural 
language processing and the application of NLP to support language 
education. He is a recipient of the MEXT scholarship for Doctoral Student.
}

\bioauthor[:]{Makoto Shishido}{%
is currently a professor of English and 
Educational Technology at the School of System Design and Technology, Tokyo 
Denki University, Japan. He earned his Ph.D. at Tohoku University. His 
current research interests include developing e-learning materials with 
speech recognition and artificial intelligence and assessing their 
effectiveness.
}

\end{biography}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\biodate



\end{document}

