\documentclass[english]{jnlp_3.3}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{hangcaption_jnlp}
\usepackage{array}
%%%%\usepackage{udline}
%%%%\setulminsep{1.2ex}{0.2ex}
%%%%\let\underline=\ul
\allowdisplaybreaks

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
%%%%\usepackage{bm}
%%%%\usepackage{here}
%%%%\usepackage{amsmath}
%%%%\usepackage{amssymb}
%%%%\usepackage{amsfonts}
\usepackage{multirow}
\usepackage{arydshln}
%%%%\usepackage{enumitem}
%%%%\usepackage{CJKutf8}
%%%%\usepackage{color}
%%%%\usepackage[utf8]{inputenc}
\usepackage{tikz}
%%%%\usepackage{multirow,makecell}
%%%%\usepackage{makecell}
%%%%\renewcommand\theadalign{bc}
%%%%\renewcommand\theadfont{\bfseries}
%%%%\renewcommand\theadgape{\Gape[4pt]}
%%%%\renewcommand\cellgape{\Gape[4pt]}
%%%%\usepackage{setspace}
\def\tabref#1{Table~\ref{#1}}
\def\figref#1{Figure~\ref{#1}}
\def\equref#1{Equation~(\ref{#1})}
\def\secref#1{Section~\ref{#1}}
\def\subref#1{Subsection~\ref{#1}}
\def\appref#1{Appendix~\ref{#1}}
%%%%\def\revise#1{#1} %% turn off the emphasis
\def\MLE{MLE}
\def\RLDisREP{RL-D$_{{\rm REP}}$}
\def\RLDisDROP{RL-D$_{{\rm DROP}}$}
\def\RLeREP{RL-eREP}
\def\RLRecall{RL-Recall}
\def\RLGLEU{RL-GLEU}
\def\RLGLEUDisREP{RL-GLEU-D$_{{\rm REP}}$}
\def\RLGLEUDisDROP{RL-GLEU-D$_{{\rm DROP}}$}
\def\DisREP{D$_{{\rm REP}}$}
\def\DisDROP{D$_{{\rm DROP}}$}
\def\DisREPDROP{D$_{{\rm REP+DROP}}$}
\newcommand{\dashedline}[1]{%
\tikz[baseline=(todotted.base)]{
\node[inner sep=1pt,outer sep=0pt] (todotted) {#1};
\draw[densely dashed] (todotted.south west) -- (todotted.south east);
}%
}%
\newcommand\blfootnote[1]{%
\begingroup
\renewcommand\thefootnote{}\footnote{#1}%
\addtocounter{footnote}{-1}%
\endgroup
}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{28}
\Number{3}
\Month{September}
\Year{2021}
%%%%\Issuetitle{「　　　　」特集号}


\received{2020}{12}{28}
\revised{2021}{4}{7}
%%%%\rerevised{2006}{5}{6}
\accepted{2021}{5}{11}

\setcounter{page}{751}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Neural Text Generation with Artificial Negative Examples to Address Repeating and Dropping Errors} 

\eauthor{Keisuke Shirai\affiref{Author_1} \and Kazuma Hashimoto$^*$\affiref{Author_2} \and Akiko Eriguchi$^*$\affiref{Author_3} \and Takashi Ninomiya\affiref{Author_4} \and Shinsuke Mori\affiref{Author_5}}

\eabstract{%
Neural text generation models that are conditioned on a given input (e.g., machine translation and image captioning) are typically trained through maximum likelihood estimation of the target text. However, models trained in this manner often suffer from various types of errors when making subsequent inferences. In this study, we propose suppressing an arbitrary type of error by training the text generation model in a reinforcement learning framework; herein, we use a trainable reward function that can discriminate between references and sentences, containing the targeted type of errors. We create such negative examples by artificially injecting the targeted errors into the references. In the experiments, we focus on two error types; repeated and dropped tokens in model-generated text. The experimental results demonstrate that our method can suppress generation errors, and achieves significant improvements on two machine translation and two image captioning tasks.
}
\ekeywords{Machine Translation, Image Captioning, Discriminator, Negative Example}

\headauthor{Shirai et al.}
\headtitle{Neural Text Generation with Artificial Negative Examples}

\affilabel{Author_1}{}{Graduate School of Informatics, Kyoto University}
\affilabel{Author_2}{}{Salesforce Research}
\affilabel{Author_3}{}{Microsoft Research}
\affilabel{Author_4}{}{Graduate School of Science and Engineering, Ehime University}
\affilabel{Author_5}{}{Academic Center for Computing and Media Studies, Kyoto University}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}
\label{sec-intro}

Conditional neural text generation models are expected to generate human-readable text that accurately describes information from a given source \cite{sutskever2014sequence,vinyals2015show}.~\blfootnote{$^*$Work performed while the authors were at the University of Tokyo.} These models are trained in a supervised fashion, by providing them with ground-truth symbols \cite{williams1989learning}. Conversely, when making inferences (i.e., during testing), the models usually generate text in a left-to-right fashion; notably, these text generation models suffer from various generation errors. For example, the models unnecessarily repeat tokens, and will drop or lose informative tokens. We refer to these two error types as {\it repeating} and {\it dropping errors}. In this study, our goal is to design a training framework that could explicitly suppress a specific type of generation error.

We focus on a reinforcement learning (RL) framework proposed by \citeA{ranzato2016sequence}, and incorporate a reward function to penalize erroneous text generation during training. The RL framework has recently been applied to several text generation tasks, and research has been conditioned on the design of task-oriented reward functions \cite{wu2016google,zhang2017sentence,rennie2017self}. More recent studies \cite{dai2017towards,gu2018neural} have proposed training a discriminative reward function (i.e., a discriminator) in generative adversarial network frameworks. However, these reward functions are not designed to handle a specific type of generation error.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1
\begin{figure}[b]
\begin{center}
\includegraphics{28-3ia1f1.pdf}
\end{center}
\caption{The overview of our method.}
\label{fig-overview}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

To answer our research question, we propose a new RL framework that suppresses an arbitrary type of generation error. \figref{fig-overview} illustrates the overview of our method. First, we train a reward function that discriminates between references and sentences, containing the targeted type of error. To train such a discriminator, we introduce artificially generated negative examples by injecting the specific type of error into the references (\figref{fig-overview} (a)). The trained discriminator is then expected to capture the targeted errors according to \citeA{ranzato2016sequence}, a text generation model learns to generate text while suppressing the specified error type (\figref{fig-overview} (b)). In this study, we consider two error types: repeating and dropping errors. We provide examples of these error types in a translation task in \tabref{tab-error-examples}. We demonstrate that our method can suppress these errors and improve the generation performance on two tasks each for translation and captioning.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[t]
\input{01table01.tex}
\hangcaption{Error generation examples of neural text generation model in a Japanese-to-English translation task. The tokens with a solid line are repeated by the generation model. The tokens with a dashed line are dropped from a source sentence at the decoding step.}
\label{tab-error-examples}
\vspace{-0.5\Cvs}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\newpage
Our contributions are three-fold.
\begin{itemize}
\item We propose a novel RL framework that suppresses text generation errors by specifically targeting an error type. We show that our method can suppress two types of generation errors: repeating and dropping errors.
\item Our method can be expansively applied to the existing RL framework using other functions, such as GLEU, to further boost its generation performance. 
\item Our analyses show that the proposed method functions more effectively then traditional methods do, as the training example size becomes smaller and more generation errors appear. 
\end{itemize}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Neural Text Generation}
\label{sec-generation}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Maximum likelihood training}

In conditional neural text generation models, an encoder encodes the source-side information $s$ (e.g., a source sentence in machine translation \cite{sutskever2014sequence} or an image in image captioning \cite{vinyals2015show}) into an intermediate representation $h^s$, and then, a decoder is trained to generate a reference $r = (r_1, r_2, \cdots, r_n)$ conditioning on $h^s$. To train the model in a supervised manner, we follow the maximum likelihood estimation (MLE) objective, as
\begin{equation}
L_{\textrm{MLE}} = -\sum_{j=1}^{n} \log p (r_j | r_{< j}, s).
\label{eq-mle}
\end{equation} 
In this study, we focus on two generation tasks: machine translation and image captioning.

At the time of inference, the text generation model generates a target-side sentence, $t = (t_1, t_2, \cdots, t_{n'})$. When generating the $j$-th token, the model follows a categorical distribution, $p(t_j | t_{<j}, s)$, to generate a token. To generate a better sentence, we use various decoding techniques, including beam search.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Reinforcement learning}
\label{subsec-rl}

Next, we describe the RL framework proposed in \citeA{ranzato2016sequence}. When training the text generation model in the RL framework, we consider the generation model as an agent and the categorical distribution at the $j$-th decoding step, $p(t_j | s, t_{<j})$, as a policy; here, $t_j$ is the $j$-th token generated by the model. We then choose a token from the categorical distribution as an action. After generating a complete target sentence, $t = (t_1, t_2, \cdots, t_m)$, REINFORCE \cite{williams1992simple} was used to define the loss function, as follows.
\begin{equation}
L_{\textrm{RL}} = -\sum_{j=1}^{m} \{ \log p(t_j | s, t_{<j}) (R(s, t) - b (s, t_j)) \},
\label{eq-rl}
\end{equation}
where $R$ is a reward function, and $b$ is a baseline network \cite{sutton2018reinforcement} that is used to reduce the variance of the gradients.\footnote{Based on \citeA{ranzato2016sequence}, we used a linear regression model as the baseline network.} However, using only \equref{eq-rl} causes the training to become unstable \cite{wu2016google}. Thus, the following joint loss function was used.
\begin{equation}
\lambda_{\text{MIXED}} \ L_{\textrm{MLE}} + (1 - \lambda_{\text{MIXED}}) \ L_{\textrm{RL}},
\label{eq-mixed}
\end{equation}
where $\lambda_{\text{MIXED}}$ is a hyperparameter that controls the strength of the two signals.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Generation errors}

Neural text generation models suffer from various types of errors, such as repeating, dropping, grammatical, and mis-ordering errors. Among them, repeating and dropping errors have been most frequently addressed in previous research \cite{mi2016coverage,malaviya2018sparse,holtzman2020curious,welleck2020neural}. As a preliminary experiment, we trained a neural machine translation model with the MLE objective on the WAT'15 Workshop on Japanese-to-English translation tasks \cite{nakazawa2016aspec}; additionally, we manually checked $100$ examples that were randomly sampled from the translation results on the test dataset. We found that $17$ sentences contained repeating errors, wherein some tokens were repeated unnecessarily, and $13$ sentences contained dropping errors, wherein some tokens were dropped from their reference sentences.\footnote{These $100$ translation examples are sampled from the RNN-based NMT (RNMT) result in \secref{subsec-results-dis}. The error examples in \tabref{tab-error-examples} were sampled from these translation results.} We also observed that the model trained with the widely used RL approach in \citeA{wu2016google} improve the BLEU scores \cite{papineni2002bleu}, but did not specifically suppress the above two types of errors.\footnote{Specific details are provided in \secref{subsec-results-gleu}.} Based on these observations, we aimed to improve the generation performance of the text generation model by suppressing these two common types of errors. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Approach}

We propose to suppress the targeted type of errors using the RL framework. Our main objective was to design an $R$ in \equref{eq-rl} that provides negative rewards to the generated text, if the text contains the targeted type of errors, and positive rewards to the text otherwise. We used a discriminator to instantiate such an $R$ function. We trained the discriminator by taking references as positive examples and erroneous sentences as negative examples. We prepared these negative examples by artificially injecting the targeted errors into the references. We describe the negative examples in \secref{subsec-ane} and the discriminator in \secref{subsec-dis}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Artificial negative examples}
\label{subsec-ane}

Our aim was to prepare a discriminator that is capable of specifically focusing on one error type. The discriminator requires negative examples, such that they always contain the targeted type of error. We created such negative examples directly from the references using an error-generating function, $e = C_{type}(r)$; here, $e$ is a sentence containing an error of a specific {\it type}, and $r$ is a reference. We designed $C_{type}$ depending on the error type on which we focused. We refer to $e$ as an artificial negative example (ANE). 

In this study, we designed two types of ANEs to handle repeating and dropping errors. \tabref{tab-examples-ane} shows a reference sentence and its negative examples for each of the two types.\footnote{Although the tokens were split at a word-level in this example, we applied our method at the subword-level in our experiments.} In the following section, we describe the design of the ANEs.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%\begin{description}[style=unboxed,leftmargin=0cm]
\paragraph{Artificial repeating sentence}
We created ANEs for repeating errors by modifying the reference sentences to repeat their tokens. In general, such repeated tokens do not always appear consecutively in model-generated sentences; therefore, we propose the $C_{type}$ function as follows. Given a reference $r$ with length $n$, $C_{repeat}(r)$ returns an artificial repeating sentence by duplicating $i$ consecutive tokens, starting from the $j$-th token, at $k$ randomly selected positions. We randomly chose $i$ from $(1, 2, \cdots, m_{\textrm{rep}})$, $j$ from $(1, 2, \cdots, n-i+1)$, and $k$ from $(1, 2, \cdots, n_{\textrm{rep}})$ for each example; here, $m_{\textrm{rep}}$ is the maximum number of consecutive tokens, and $n_{\textrm{rep}}$ is the maximum number of duplications. The $k$ random positions were selected from $(1, 2, \cdots, j-1, j+i-1, \cdots, n-1, n)$ to avoid breaking the original consecutive tokens. In \tabref{tab-examples-ane}, three consecutive tokens ``by the company,'' ($i=3$) starting from the $5$-th token ($j=5$), were repeated twice ($k=2$), and were inserted after the $7$-th token, ``company,'' and the $9$-th token, ``keeps,'' of the reference. We set the hyperparameters $(m_{\textrm{rep}},n_{\textrm{rep}}) = (4,4)$. We instead set $m_{\textrm{rep}} = n$ if $n$ was smaller than $n_{\textrm{rep}}$. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% tab le 2
\begin{table}[t]
\input{01table02.tex}
\hangcaption{Examples of artificial negative examples. Three consecutive tokens ``by the company'' are repeated in the repeating example, while they are dropped in the dropping example.}
\label{tab-examples-ane}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Artificial dropping sentence}
We created ANEs for dropping errors by modifying the reference sentences to drop consecutive tokens. Given a reference $r$ with length $n$,  $C_{drop}(r)$ returns an artificial dropping sentence by dropping $i$ consecutive tokens, starting from the $j$-th token. We randomly chose $i$ from $(1, 2, \cdots, m_{\textrm{drop}})$ and $j$ from $(1, 2, \cdots, n-i+1)$ for each example, where $m_{\textrm{drop}}$ is the maximum number of consecutive tokens. In the example that is presented in \tabref{tab-examples-ane}, three consecutive tokens ($i=3$), starting from the $5$-th token ($j=5$), were dropped. We set the hyperparameter $m_{\textrm{drop}} = 4$. This setup allowed us to drop ${i}$ randomly chosen consecutive tokens ($i = 1, 2, \cdots, 4$). We instead set $m_{\textrm{drop}} = n$ if $n$ was smaller than $m_{\textrm{drop}}$ and larger than $2$. $C_{drop}$ returns an end-of-sentence token if $n$ equals to $1$. 
%%%%\end{description}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Discriminator}
\label{subsec-dis}

We trained the discriminator using references and their ANEs. Our discriminator was a binary classifier that accepted a pair of source $s$ and its target sentence (either $r$ or $e = C_{type}(r)$) as the input and a real value in [0, 1] as the output. We minimized the following loss function as follows.
\begin{equation}
L_{\textrm{DIS}} 
=  -\mathbb{E}_{s, r}
  [\log \text{D}(s, r)] - \ \mathbb{E}_{s, e}
[\log (1 - \text{D}(s, e))],
\label{eq-dis}
\end{equation}
where $\text{D}$ denotes the discriminator. During the discriminator training, we called $C_{type}(r)$ every time we processed $r$ in a mini-batch. Once the discriminator training was completed, we used the trained discriminator $\text{D}$ as $R(s, t)$ in \equref{eq-rl} by freezing its parameters, such that $R(s, t)$ could output a reward value for a generated sentence $t$ to train the text generation models.

Our discriminator $\text{D}$ comprises two types of encoders: source-side and target-side. The source-side encoder encodes source-side information $s$ into a fixed-size vector $h^s$. The target-side encoder receives $h^s$, and encodes the target sentence $t = (t_1, t_2, \cdots, t_n)$ into a sequence of representations $H^t = (h^t_1, h^t_2, \cdots, h^t_n)$. Once $H^t$ is calculated, we compute $\hat{h}^t$ as $\hat{h}^t = maxpool(H^t)$. The discriminator finally obtains output $y$, as
\[
y = f_{\text{sigmoid}}(W_o f_{\text{ReLU}}(W_h \hat{h}^t + b_h) + b_o).
\]
Here, $W_h \in \mathbb{R}^{d_h/2 \times d_h}$, $b_h \in \mathbb{R}^{d_h/2}$, $W_o \in \mathbb{R}^{1 \times d_h/2}$, and ${b_o \in \mathbb{R}^1}$ are learnable parameters, and $d_h$ is the dimension of $h^t$. $f_{\text{ReLU}}$ is the rectified linear unit (ReLU), and $f_{\text{sigmoid}}$ is the logistic sigmoid function.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experimental Settings}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Datasets}

Regarding machine translation, we conducted experiments on Japanese-to-English (Ja-En) and German-to-English (De-En) tasks. For the Ja-En task, we used the Asian scientific paper excerpt corpus (ASPEC) from WAT'15. Specifically, we used \texttt{train-1.txt} and \texttt{train-2.txt} for training, \texttt{dev.txt} for development, and \texttt{test.txt} for testing, according to \citeA{hashimoto2019accelerated}. Both Japanese and English sentences were preprocessed as recommended in WAT'15.\footnote{\url{http://lotus.kuee.kyoto-u.ac.jp/WAT/WAT2015/baseline/dataPreparationJE.html}} For the De-En task, we used the datasets provided by WMT'16\footnote{\url{http://www.statmt.org/wmt16/translation-task.html}} as well as all parallel corpora (\texttt{Europarl v7}, \texttt{Common Crawl corpus}, and \texttt{News Commentary v11}) for training; NewsTest2013 for development; and NewsTest2014 for testing.

Regarding image captioning, we conducted experiments on two datasets: MS COCO \cite{lin2014microsoft} and Flickr30K \cite{plummer2015flickr30k}. For training, developing, and testing, we followed the splits provided by \citeA{karpathy2015deep}.

We used {\tt SentencePiece} \cite{kudo2018sentencepiece} to tokenize sentences and build vocabularies. We empirically chose a vocabulary size of $16,000$ for each language in the Ja-En translation task; $16,000$ for both languages in the De-En translation task, because these languages share an alphabet \cite{sennrich2016neural}; and $2,000$ for the MS COCO and Flickr30K captioning tasks. The vocabulary also contained three special tokens: $\langle$s$\rangle$, which signaled the beginning, $\langle$/s$\rangle$, which signaled the end of a sentence, and $\langle$unk$\rangle$ for out-of-vocabulary tokens. \tabref{tab-datasets} shows the statistics regarding the different datasets. During training in machine translation, we removed all empty sentences and any sentence pairs, whose maximum length was longer than $80$, from the training dataset.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[t]
\input{01table03.tex}
\caption{Dataset statistics and vocabulary sizes \vline \ \it{V} \vline.}
\label{tab-datasets}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Models}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Machine translation}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Text generation model}
For the translation tasks, we used two types of translation models: RNMT model \cite{bahdanau2015neural} and the Transformer  \cite{vaswani2017attention}.
Our RNMT model is an attention-based NMT model with a $2$-layer bidirectional long short-term memory (LSTM) \cite{hochreiter1997long} encoder and a $2$-layer unidirectional LSTM decoder. 
We used the attention mechanism proposed by \citeA{bahdanau2015neural}.
The hidden state size and embedding size were both set to $512$.

Our Transformer comprises a $6$-layer encoder and $6$-layer decoder. The hidden state size and number of heads were set to $512$ and $8$, respectively. To add positional information, we used a positional encoding technique with sine and cosine functions, as proposed by \citeA{vaswani2017attention}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Discriminator}
For the discriminator, we used a 2-layer unidirectional LSTM for the source-side and target-side encoders, with $d_h = 512$.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Image captioning}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Text generation model}
For the image captioning tasks, we used a simple show-and-tell model \cite{vinyals2015show}. We used ResNet-152 \cite{he2016deep}, which was pre-trained on an ImageNet classification dataset \cite{russakovsky2015imagenet}, as an encoder by freezing its model parameters to only extract features, and used a $512$-dimensional $1$-layer unidirectional LSTM as a decoder. Once an image feature $f^i \in \mathbb{R}^{2048}$ is extracted, the decoder state $h^t_0$ is initialized as $h^t_0 = W_i f^i + b_i$; here, $W_i \in \mathbb{R}^{512 \times 2048}$ is a weight matrix, and $b_i \in \mathbb{R}^{512}$ is a bias vector.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Discriminator}
For the discriminator, we projected the feature vector $f^i$ into a fixed-size vector $h^s$ as $h^s = W_D f^i + b_D$, where the weight matrix $W_D \in \mathbb{R}^{512 \times 2048}$ and a bias vector $b_D \in \mathbb{R}^{512}$ are learnable parameters. We used a unidirectional LSTM as the target-side encoder, with $d_h = 512$.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Training strategies}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Training discriminators}

We used Adam \cite{kingma2015adam} with an initial learning rate of $1.0 \times 10^{-3}$ and a weight decay at a rate of $1.0 \times 10^{-6}$. We assessed its accuracy on the development dataset every $1,000$ iterations, and halved the learning rate when the accuracy worsened. The training was concluded after the learning rate was halved five times. We chose the best models based on those with the best accuracy with the development dataset. For the development dataset, we created one negative example for each reference sentence. %and used it to calculate the accuracy.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Training text generation models}

The training of the text generation models can be divided into two steps: a pre-training step with MLE loss \equref{eq-mle} and reinforcement learning step with RL loss \equref{eq-mixed} and the pre-trained model, following the steps outlined in \citeA{wu2016google}.

In the pre-training step, we used Adam with an initial learning rate of $1.0 \times 10^{-3}$ and a weight decay with a rate of $1.0 \times 10^{-6}$, unless stated otherwise. Each mini-batch contained $128$ examples. We assessed its perplexity on the development dataset every $1,000$ iterations, and halved the learning rate if the perplexity worsened. We finished the training when we halved the learning rate five times. For the Transformer, we used AdamW \cite{loshchilov2019decoupled} with a weight decay with a rate of $1.0 \times 10^{-4}$. When using AdamW, we scheduled the learning rate as follows.
\begin{equation}
lr = \begin{cases}
lr_{\text{ini}} + step \times \frac{lr_{\text{max}} - lr_{\text{ini}}}{N_{\text{wm}}} & \text{if}\ step \leq N_{\text{wm}} \\
lr_{\text{max}} \times \eta(step) & \text{otherwise} 
\end{cases},
\label{eq:cosine}
\end{equation}
where
\[
\eta(step) = 0.5 + 0.5 \times \text{cos}(\pi \times \frac{step - N_{\text{wm}}}{N_\text{wm} \times N_{\text{cl}}}). \nonumber
\]
This scheduling comprised two steps: a linear warmup step from $lr_{\text{ini}}$ to $lr_{\text{max}}$, for the first $N_{\text{wm}}$ iterations, and a cosine annealing step from $lr_{\text{max}}$ to $0$, for $N_{\text{wm}} \times N_{\text{cs}}$ iterations. 
We used empirically tuned hyperparameters as $(lr_{\text{ini}}, lr_{\text{max}}, N_{\text{wm}}, N_{\text{cs}}) = (5.0 \times 10^{-6},\ 5.0 \times 10^{-4},\ 4,000,\ 24)$. Each mini-batch contained $512$ examples for the Transformer.
For the pre-training step, we clipped the gradients \cite{pascanu2013difficulty} with a value of $1.0$, and used the label-smoothing technique \cite{szegedy2016rethinking} with a rate of $0.1$. We chose the best models based on the lowest perplexity with the development dataset.

During the reinforcement learning step, we used a stochastic gradient descent with momentum. For the translation tasks, we tuned the learning rate of each model with a momentum rate of $0.9$. For the captioning tasks, we consistently used a learning rate of $5.0 \times 10^{-2}$ with a momentum rate of $0.9$. During this step, we extracted $20,000$ iterations to fine-tune the parameters of the generation model. We continued to use the same gradient clipping and label-smoothing techniques as outlined above. Each mini-batch contained $64$ examples. In particular, we carefully tuned $\lambda_{\text{mixed}}$ in \equref{eq-mixed}.\footnote{Specifically, we searched the values in a coarse-to-fine fashion in \{0.5, 0.3, 0.1, $7.5 \times 10^{-2}$, $5.0 \times 10^{-2}$, $2.5 \times 10^{-2}$, $1.0 \times 10^{-2}$, $7.5 \times 10^{-3}$, $5.0 \times 10^{-3}$, $2.5 \times 10^{-3}$, and $1.0 \times 10^{-3}$\}.} We report the inferences of our model as generated by a beam search with a width of $10$.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Evaluation metrics}

One goal of this study is to suppress the frequency of repeating and dropping errors. To quantitatively evaluate these errors in the model's output, we used REP and DROP scores \cite{malaviya2018sparse}. In this section, we first describe these two metrics in detail, and then, introduce our task-specific metrics.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{REP score}

The REP score calculates the number of $n$-gram repetitions that are included in a model-generated sentence $t$, given its reference $r$, as follows.
\[
\text{REP}(r, t) = \frac{\sigma(t,r)}{ \sum_{w \in V} r(ww) + \sum_{s \in V^{n}_r} r(s)},
\]
where
\begin{align}
\sigma(t,r) = & \lambda_2 \underset{\hspace{0.2cm} s \in V^{n}_r, t(s) \geq 2}{\sum} \hspace{-0.25cm} \max \{0, t(s) - r(s)\} \nonumber \\
 & + \lambda_1 \sum_{w \in V} \max \{0, t(ww) - r(ww)\}. 
\label{eq-rep}
\end{align}
$V^{n}_r$ is the set of all $n$-grams included in the reference. 
$r(ww)$ and $r(s)$ indicate the frequency of consecutive $1$-grams $w$ and $n$-grams $s$ of the reference, respectively, and $t(ww)$ and $t(s)$ are those of the machine-generated sentence $t$. $\lambda_1$ and $\lambda_2$ are hypeparameters.

The REP score is defined for each $n$-gram separately; however, in this study, we propose using an extended REP (eREP) score that evaluates consecutive $1$-gram and $n$-grams ($n=2,\ 3,\ 4$).
The eREP score is calculated as follows.
\[
\text{eREP}(r, t) = \frac{\sigma(t,r)}{ \sum_{w \in V} t(ww) + \sum_{s \in V^{n}_t} t(s)},
\]
where
\begin{align*}
\sigma(t,r) = & \sum^{4}_{n=2} \lambda_n\underset{\hspace{0.2cm} s \in V^{n}_t, t(s) \geq 2}{\sum} \hspace{-0.25cm} \max \{0, t(s) - r(s)\} \nonumber \\
 & {} + \lambda_1 \sum_{w \in V} \max \{0, t(ww) - r(ww)\}.
\end{align*}
$V^{n}_t$ is the set of all $n$-grams included in the machine-generated sentence $t$. %We remove $\lambda_1$ and $\lambda_2$ in \equref{eq-rep} to 
We weight the repetitions of $n$-grams equally ($\lambda_n=1$). 
For the eREP score, a lower value is better.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{DROP score}

For the machine translation tasks, the DROP score calculates the number of source tokens that are {\it not} covered (i.e., included) in the model-generated sentences. A word alignment tool is used to identify which source tokens are aligned with their referent, and to calculate the ratio of those not aligned with the generated text (hypothesis). In other words, this study aims to evaluate how well the source information is covered, and to assess if the DROP score is defined as follows:
\[
\text{DROP}(c_{\text{ref}}, c_{\text{hyp}}) = 1 - \frac{1}{|c_{\text{ref}}|} \sum_{i \in c_{\text{ref}}} in(i),
\]
where $c_{\text{ref}}$ and $c_{\text{hyp}}$ represent the set of source token indices in the source-reference and source-hypothesis alignments, respectively. $in(i)$ is a function that returns $1$ if an index $i$ is included in $c_{\text{hyp}}$ and $0$ otherwise. For the DROP score, a lower score is better, as is the case for the eREP score.

For the captioning tasks, however, the DROP score was not applicable, because no source sentence exists. For our experiment, we instead used ROUGE$_L$ \cite{chen2015microsoft} to evaluate the number of dropping errors that were found in the model's output. In ROUGE$_L$, contrary to the DROP score, a higher score is better. During our preliminary experiment, to assess the relationship between the DROP score and ROUGE$_L$, we calculated the Pearson correlation coefficient between ROUGE$_L$ and DROP on the RNMT's WAT'15 Ja-En result. We confirmed a negative moderate correlation ($-0.430$) between these two metrics, suggesting that we can use ROUGE$_L$, instead of the DROP score, for the captioning tasks. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Task-specific metrics}

We used BLEU and METEOR \cite{malaviya2018sparse} for the translation tasks, and BLEU and CIDEr \cite{vedantam2015cider} for the captioning tasks. For the captioning tasks, we used publicly available code\footnote{\url{https://github.com/tylin/coco-caption}} to calculate the BLEU and CIDEr scores. Note that, for the captioning tasks, we report BLEU-{1,2,3,4}. % and ROUGE$_L$ in [0, 1]. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Model configurations}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%\begin{description}[style=unboxed,leftmargin=0cm]
\paragraph{-- \MLE}
is the baseline model trained by the MLE loss in \equref{eq-mle}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{-- \RLDisREP, \RLDisDROP}
are our proposed models trained by the RL framework in \equref{eq-mixed} with our proposed discriminator. 
The text generation model parameters were initialized using the \MLE baseline. \RLDisREP\ and \RLDisDROP\ were trained to suppress repeating and dropping errors with discriminators \DisREP\ and \DisDROP, respectively. 

%%%%\end{description}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Results}
\label{sec-results}

We report the scores in the range of [0, 100]. The symbol~$\dagger$ follows a score if the system produced a significant improvement against \MLE\ (p $<$ $0.05$). In our experiments, a statistical significance test was performed using the paired bootstrap resampling method \cite{koehn2004statistical}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Suppressing the targeted errors}
\label{subsec-results-dis}

In this section, we present the results obtained using the discriminator to suppress repeating and dropping errors. For the \RLDisREP\ and \RLDisDROP\ models, we chose the best models based on the scores from the development datasets that we would like to improve (e.g., the eREP score was used for \RLDisREP). 

We first present the accuracy of our discriminators because the discriminator plays a key role in our method. \tabref{tab-results-discriminator} reports the binary classification accuracy for the development datasets. \DisREP\ achieved over $96$\% accuracy for all of the tasks, whereas the accuracy of \DisDROP\ was not as high as that of \DisREP. A possible reason for this is that identifying the dropped tokens in a sentence is an inherently more difficult task than identifying the $n$-gram repetitions.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 4
\begin{table}[b]
\input{01table04.tex}
\caption{Accuracy of the discriminator on the development dataset.}
\label{tab-results-discriminator}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 5
\begin{table}[b]
\input{01table05.tex}
\caption{Results on the translation tasks.}
\label{tab-results-translation}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\tabref{tab-results-translation} shows the main results from the translation tasks. \RLDisREP\ consistently improved its eREP scores, with the exception that the MLE-based Transformer in the De-En task had much less room for improvement. \RLDisDROP\ also consistently improved its DROP scores for both the RNMT and Transformer models. When receiving rewards from the discriminator, namely \DisREP\ and \DisDROP, the text generation models learned to suppress the repeating and dropping errors. Note also that \RLDisREP\ tended to increase the BLEU and \RLDisDROP\ tended to increase the METEOR, with some exceptions. For the former, we consider that reducing the number of repetitions using \RLDisREP\ led to higher $n$-gram precision; thus, the model was able to achieve higher BLEU scores. For the latter, this is presumably because METEOR is based on unigram precision and recall, and thus, places more weight on recall than precision. Therefore, suppressing the dropping errors and restoring dropped tokens using \RLDisDROP\ contributed to the improvements in the METEOR score.

\tabref{tab-results-captioning} shows the results from the image captioning tasks. Again, note that \RLDisREP\ significantly improved the eREP scores, leading to better BLEU-{1,2,3,4} scores for both the COCO and Flickr30K tasks. Alternatively, \RLDisDROP\ did not achieve significant improvements in ROUGE$_L$. One possible reason arises from the diversity of the text in the captioning task \cite{dai2017towards}. That is, identifying whether the dropping error occurred in a sentence would be an easy task for the discriminator; however, generating such tokens would be difficult for the generation model. 
We also observed that the balancing factor $\lambda_{\text{mixed}}$ was preferred to be smaller for \RLDisREP\ than for \RLDisDROP. This indicates that the generation model relies more on the reward from \DisREP\ than from \DisDROP, which explains why \RLDisREP\ produced more expected results than \RLDisDROP\ in the captioning tasks.\footnote{For example, we considered $5.0 \times 10^{-3}$ and $5.0 \times 10^{-3}$ for \RLDisREP, and $0.1$ and $5.0 \times 10^{-2}$ for \RLDisDROP\ on the COCO and Flickr30K tasks, respectively.} 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 6
\begin{table}[t]
\input{01table06.tex}
\caption{Results on the captioning tasks.}
\label{tab-results-captioning}
%%%%\vspace{-1\Cvs}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We show examples of the generation performance of the RNMT-based models in the WAT'15 Ja-En translation task in \tabref{tab-examples-wat15}. In Example (A), \RLDisREP\ successfully suppressed the repeated tokens that were found in \MLE\ and generated a non-repetitive sentence. In Example (B), \RLDisDROP\ satisfactorily restored the dropped tokens in \MLE, and thus, generated a more informative sentence.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 7
\begin{table}[t]
\input{01table07.tex}
\hangcaption{Generation examples of the RNMT-based models on the Ja-En translation task. The underlined tokens with solid lines represent the repeating error. The underlined tokens with a dashed line are tokens \MLE\ failed to generate.}
\label{tab-examples-wat15}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 8
\begin{table}[t]
\input{01table08.tex}
\caption{Computation time on the translation tasks.}
\label{tab-computation-time}
\vspace{-0.5\Cvs}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We report the computation time of each model on the translation tasks in \tabref{tab-computation-time}. For all the experiments, we used an \texttt{Intel(R) Core(TM) i7-7700K CPU @ 4.20GHz} and a single GPU; \texttt{NVIDIA GeForce GTX 1080 Ti}. Note that the pre-training step (\MLE) required 50,000 iterations for the RNMT and 100,000 iterations for the Transformer, whereas the reinforcement learning step required 20,000 iterations for both models.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Incorporating an off-the-shelf reward function}
\label{subsec-results-gleu}

In \secref{subsec-results-dis}, we report that our proposed discriminator can suppress the targeted type of errors. We investigate how the performance of existing reward functions compare with our discriminators to further improve the generation performance by incorporating them. Taking GLEU \cite{wu2016google} as an example of existing rewards, we propose the following joint reward function.
\pagebreak
\begin{align}
R(s, t)  = & \lambda_{\text{RL}} R'(s, t) \nonumber \\
&   {}  + (1 - \lambda_{\text{RL}}) GLEU(t, r),
\label{eq-reward}
\end{align}
where $R'$ is one of our reward functions, and $GLEU$ is the GLEU score. GLEU is known to be effective in improving the BLEU score \cite{wu2016google}. $GLEU(t, r)$ calculates the minimum of the generated sentence $t$'s $n$-gram precision and recall against the reference $r$. $\lambda_{\text{RL}}$ is a hyperparameter that controls the strength of the two signals. In this section, we use \RLGLEU\ to refer to the model that uses only the GLEU as the reward and \RLGLEUDisREP\ (or \RLGLEUDisDROP) to refer to the model that uses both the GLEU and its corresponding discriminator. We chose the best models based on the best BLEU scores from the development datasets.

\tabref{tab-results-translation-gleu} shows the results from the translation tasks. In the Ja-En task, \RLGLEU\ contributed to the improvement in the BLEU, DROP, and METEOR scores. Because the GLEU computed a recall against the reference, \RLGLEU\ consequently generated more informative tokens and improved those scores. Thus, \RLGLEUDisDROP\ showed less improvement on its DROP score compared to previous experiments. \RLGLEUDisREP\ further improved the eREP, particularly when \RLGLEU\ deteriorates the eREP score. In the De-En task, the Transformer-based \RLGLEUDisREP\ and \RLGLEUDisDROP\ showed almost the same results as \RLGLEU. This might be due to the large number of training examples; we will discuss the possible effect of the training data size in \secref{subsec-results-scale}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 9
\begin{table}[b]
\input{01table09.tex}
\hangcaption{Results of the joint models with GLEU on the translation tasks. Note that the \MLE\ results are the same in \tabref{tab-results-translation}.}
\label{tab-results-translation-gleu}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\tabref{tab-results-captioning-gleu} shows the results from the image captioning tasks. In both the COCO and Flickr30K tasks, \RLGLEU\ significantly improved the BLEU and CIDEr scores, but generated more repetitions as indicated by the eREP scores. \RLGLEUDisREP\ achieved further improvements on the BLEU scores by suppressing the occurrence of repeating errors. \RLGLEUDisDROP, alternatively, did not improve the performance of \RLGLEU. This may be because it is difficult to suppress dropping errors even when using both the GLEU and discriminator's signals, as discussed in \secref{subsec-results-dis}. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 10
\begin{table}[t]
\input{01table10.tex}
\hangcaption{Results of the joint models with GLEU on the captioning tasks. Note that the \MLE\ results are the same in \tabref{tab-results-captioning}.}
\label{tab-results-captioning-gleu}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Comparison with related studies}

In the field of machine translation, there are two domains of research that share similarities with our research direction: (i) methods based on coverage and (ii) methods based on the generative adversarial network (GAN). The coverage-based method \cite{mi2016coverage,tu2017context,malaviya2018sparse} uses attention history to reduce the number of repeating and dropping errors. Alternatively, the GAN-based method \cite{gu2018neural,yang2018improving} utilizes adversarial training with a discriminator to train the generation model, to generate more natural sentences. In this section, we conduct further experiments to compare the performance of these alternate methods with that of ours. In this section, we focus on the RNMT model.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Comparison with the coverage-based method}

We first conducted experiments using the coverage-based model. We decided to use the coverage vector \cite{tu2017context}, which stores the attention history and uses it in the decoding process to more effectively utilize untranslated source words. We used neural network-based coverage, which uses RNNs to model the coverage vector, and set the coverage dimension as $10$ following the procedures outlined in \cite{tu2017context}. Although \citeA{tu2017context} used a gated recurrent unit as an activation function, we used LSTM, and empirically confirmed that LSTM performs well in this context. We refer to this model as CovVec. 

\tabref{tab-results-translation-additional} shows the results of this comparison. For both the two translation tasks, we can see that CovVec successfully improved the eREP and DROP scores from the \MLE\ results, as expected. Compared to \tabref{tab-results-translation}, the eREP scores were close to those of \RLDisREP\ ($2.53$ in the Ja-En task and $0.78$ in the De-En task), and the drop scores are reasonably higher than those of \RLDisDROP\ ($14.63$ in the Ja-En task and $3.59$ in the De-En task). However, the CovVec did not produce any significant improvements in the BLEU and METEOR scores in our experiments, although \RLDisREP\ in the Ja-En task achieved an improvement in BLEU. We consider that this improvement results from the following fact. By fine-tuning the parameters of \MLE, our model was able to concentrate on suppressing the targeted type of errors for erroneous sentences, and continue to generate the same sentences for other, non-problematic sentences. When we checked the translations of \RLDisREP\ and \MLE, we found that \RLDisREP\ tended to generate nearly identical translations of \MLE\ for some source sentences, supporting our view.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 11
\begin{table}[t]
\input{01table11.tex}
\hangcaption{Results of the coverage-based and GAN-based models. Note that the \MLE\ results are the same in \tabref{tab-results-translation}.}
\label{tab-results-translation-additional}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Comparison with the GAN-based method}

Next, we conducted experiments on the GAN-based model. We decided to use the Gumbel-Greedy decoding (GGD) method \cite{gu2018neural}, which bridges the generation model and discriminator using the Gumbel-Softmax estimator \cite{jang2017categorical}. The hyperparameters ($N_g$, $N_d$) in \cite{gu2018neural} were set to ($1$, $1$). We refer to this model as GGD. 

The results of this comparison are shown in \tabref{tab-results-translation-additional}. For BLEU and METEOR, the GGD consistently improved both scores from the \MLE\ results. Furthermore, the GGD had worse the eREP scores, but improved the DROP scores in the Ja-En task; conversely, in the De-En task, the GGD improved both scores. This indicates that the GGD generation model did not always learn to suppress both repeating and dropping errors. One possible reason for this is that the type of error that the discriminator handled can differ meaningfully, as the frequency of each error type included in machine-generated sentences changes depending on the task and generation model.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Varying training set size}
\label{subsec-results-scale}

We have confirmed that our proposed method performs effectively to suppress the targeted type of errors on the WAT'15 Ja-En translation task, as well as the COCO and Flickr30K captioning tasks, but is less effective in the WMT'16 De-En task, especially during joint training with the GLEU. One possible reason is that the size of the training dataset in the De-En translation task was sufficiently large to train the language model well and suppress the targeted errors caused by data sparseness. We hypothesize that our method would be more effective when the training dataset is small; as in such a situation, the traditional language model would not be trained well, and would produce more errors. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 2
\begin{figure}[b]
\vspace{-1\Cvs}
\begin{center}
\includegraphics{28-3ia1f2.pdf}
\end{center}
\caption{The results on the WMT'16 De-En translation task when varying the training size exponentially.}
\label{fig-scale}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

To verify this hypothesis, we conducted experiments by varying the training dataset size exponentially (1/1, 1/2, 1/4, 1/8) for the WMT'16 De-En translation task. For the sake of simplicity, we changed only the training dataset size, and set the same hyperparameters as those used in \secref{subsec-results-gleu}. \figref{fig-scale} shows the results of this comparison. Note that the 1/1 case is the same as that in \secref{subsec-results-gleu}. \figref{fig-scale} (a), (b), and (d) show that the results meet our expectation; namely when the size of the training dataset is smaller, our strategy is more effective with respect to the eREP, DROP, and BLEU scores. In \figref{fig-scale} (c), in contrast, \RLGLEUDisREP\ is less effective because the Transformer language model appears to be sufficiently strong to suppress most of the repetitions, even with the smallest training dataset.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Sampling parameters for generating artificial negative examples}

In the previous sections, we trained the discriminator with ANEs, and negative examples were created based on the parameters $(i, j, k)$ described in \secref{subsec-ane}. However, there is no guarantee that the generated errors matched the distribution nor tendency of the errors produced by the generation model in practice; thus, there is a chance that the default parameter settings for the negative examples might be inappropriate, and tuning the parameters for each model might obtain further improvement. To examine this, we conducted an additional experiment wherein we trained the discriminator, and used ANEs with the parameters sampled from the statistics of the actual generation errors.

We tested this on the WAT'15 Ja-En task using the RNMT model. First, we sampled $100$ examples from the translation results of the development dataset. We then examined the phrase length, starting index of the errors, number of repetitions or droppings of the phrase, and their parts-of-speech. We found $10$ repeating and $28$ dropping errors. The statistics of these errors are shown in \tabref{tab-sampled-stats}. We then trained the discriminator with ANEs by sampling the phrase length $i$ in \secref{subsec-ane} from the statistics. For the repeating errors, we also sampled the number of repetitions ($k$ in \secref{subsec-ane}) from the statistics. For the starting index $j$ in \secref{subsec-ane}, we continued to sample the value uniformly, as there was no tendency or regularity for the starting index of the errors. We present the experimental results of the generation models trained with the discriminators in \tabref{tab-results-sampled-stats}. We add ``sampling'' to the model names to indicate the enhanced models introduced in this section. Compared with the results of \RLDisREP\ and \RLDisDROP\ from \tabref{tab-results-translation}, training the discriminator with ANEs specially tuned for the model has little effect on the overall performance of the generation model. One possible reason for this is that the discriminator trained with the negative examples without sampling parameters already had the ability to capture the errors of the generation models, and thus, performed sufficiently well to discriminate the errors during the reinforcement learning step.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 12
\begin{table}[t]
\input{01table12.tex}
\hangcaption{Statistics of errors sampled from $100$ translation results of the RNMT model on the WAT'15 Ja-En task. The number in the parentheses represents the frequency.}
\label{tab-sampled-stats}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 13
\begin{table}[t]
\input{01table13.tex}
\caption{Results}
\label{tab-results-sampled-stats}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related work}
\label{sec-results-discussion}

Training neural text generation models with a discriminator has recently been studied in GAN-based methods \cite{dai2017towards,gu2018neural,yang2018unsupervised}. 
These studies utilized the generative adversarial framework \cite{goodfellow2014generative,arjovsky2017wasserstein} to train sentence generation models to generate more natural, human-like sentences. Although our framework can also be considered as such a generator-discriminator framework, there are two points that differentiate our framework from the GAN-based methods. First, our generator can focus on suppressing a targeted type of error, as the discriminator learns to discriminate that error type from the references with artificially generated erroneous sentences. The other is that our ANEs definitely contain errors, whereas the machine-generated sentences in the GAN-based methods can include correct sentences, as they are sampled from the generation model.

Several studies have used negative examples to train sentence generation model. Focusing on the use of repetitions, \citeA{welleck2020neural} proposed the technique of unlikelihood training, wherein the training objective was to decrease the probability of generating the token that had appeared in the previous context. %Our method also can focus on the repetitions, but its target error type is not limited to the repeating errors.
\citeA{he2020negative} collected negative examples from the model generations, and trained the model to no longer generate them. 
One advantage of our method is that our discriminator is trained independent of the sentence generation model; consequently, our method does not require any extra steps to collect negative examples from the sentence generation model.

Coverage-based methods in machine translation have also aimed to suppress the occurrence of repeating and dropping errors by focusing on the coverage information regarding the source words \cite{mi2016coverage,tu2017context,malaviya2018sparse}. 
In abstractive summarization, \citeA{suzuki2017cutting} attempted to address repeating errors by controlling the output words with the estimated upper-bound frequency of words to be generated. 
Although our method is not limited to these two error types, there are two obstacles to overcome for addressing different types of errors than the ones assessed here. First is the problem of classifying the error type, which can be as difficult as the actual generation of outputs containing the various types of errors with a certain frequency. Hence, detecting and classifying all error types would seem to be a difficult task that is likely to fail. The other main obstacle is designing the corresponding ANEs for these other error types. This would also be difficult if the errors are difficult to produce based on the reference sentences that we relied on in this study.

Automatic post-editing aims to edit machine-translated sentences to fix errors, writing styles, and so on \cite{freitag2019ape,gu2019levenshtein}. The task is similar to ours, in that it considers how model-generated sentences should be improved. However, there are two significant differences. First, researchers studying automatic post-editing typically use a machine translation model and post-editing model independently, whereas we consistently trained one model to generate fewer erroneous sentences. Second, our model explicitly focused on one specific generation error type, which was not feasible in the previous work.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}

We propose a reinforcement learning-based method that directly suppresses a targeted type of error, unlike conventional methods. Our method uses a discriminator that captures a specific type of error, and this discriminator is trained with artificially generated negative examples. We empirically demonstrate that our approach can effectively reduce the number of repeating and dropping errors. In future work, it will be of interest to explore the application of our method to source-side, unannotated data. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.6}
\bibliography{01refs}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Keisuke Shirai}{Keisuke Shirai received his B.S. from Ehime University in 2017 and M.S. from Kyoto University in 2020. He is currently a Ph.D student in the Graduate School of Informatics at Kyoto University. His research interests include natural language processing, reinforcement learning, and symbol grounding.
}

\bioauthor[:]{Kazuma Hashimoto}{Kazuma Hashimoto received his B.S., M.S., and Ph.D from the University of Tokyo in 2013, 2015, and 2018, respectively. He is currently a researcher at Salesforce Research. His research interests include natural language processing both from academic and applied sides.
}

\bioauthor[:]{Akiko Eriguchi}{Akiko Eriguchi received her B.S. and M.S. from Ochanomizu University in 2013 and 2015, respectively, and Ph.D from the University of Tokyo in 2018. She is currently a research scientist at Microsoft Corporation. Her research interests include natural language processing and machine learning. She is a member of the Japan Association for Natural Language Processing and the Association for Computational Linguistics.
}

\bioauthor[:]{Takashi Ninomiya}{Takashi Ninomiya received his B.S., M.S., and Ph.D from the University of Tokyo in 1996, 1998, and 2001, respectively. He is currently a professor at the Graduate School of Science and Engineering, Ehime University. His research interests include natural language processing and computational linguistics. He is a member of the Japan Association for Natural Language Processing, the Information Processing Society of Japan, the Japanese Society of Artificial Intelligence, the Institute of Electronics, Information and Communication Engineers, the Asia-Pacific Association for Machine Translation, and the Database Society of Japan.
}

\bioauthor[:]{Shinsuke Mori}{Shinsuke Mori received his B.S., M.S., and Ph.D from \linebreak
Kyoto University in 1993, 1995, and 1998, respectively. After joining the Tokyo Research Laboratory of International Business Machines in 1998, he studied language modeling and its application to speech recognition and language processing. He is currently a professor at the Academic Center for Computing and Media Studies, Kyoto University. His research interests include natural language processing, spoken language processing, human-computer interaction, and multimedia integration. He is a member of the Japan Association for Natural Language Processing, and the Information Processing Society of Japan, and the Database Society of Japan.
}

\end{biography}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\biodate




\end{document}

