\documentclass[english]{jnlp_3.5}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{hangcaption_jnlp}
\usepackage{array}
%%%%\usepackage{udline}
%%%%\setulminsep{1.2ex}{0.2ex}
%%%%\let\underline=\ul
\allowdisplaybreaks


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
\usepackage{bm} % edited by hmino
%%%%\usepackage{bbm} % edited by hmino
%%%%\usepackage{multirow} % edited by hmino
%%%%\usepackage{amssymb}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{29}
\Number{4}
\Month{December}
\Year{2022}
%%%%\Issuetitle{Special Issue: XXXXXXXX}

\received{2022}{3}{31}
\revised{2022}{7}{22}
\accepted{2022}{8}{20}

\setcounter{page}{1082}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Lexically Constrained Knowledge Distillation\\ for Neural Machine Translation}
\eauthor{Hideya Mino\affiref{Author_1}\affiref{Author_2} \and Kazutaka Kinugawa\affiref{Author_1} \and Hitoshi Ito\affiref{Author_1} \and Isao Goto\affiref{Author_1} \and Ichiro Yamada\affiref{Author_1} \and Takenobu Tokunaga\affiref{Author_2}}
\eabstract{
Knowledge distillation is a representative approach in neural machine translation (NMT) for compressing a large model into a lightweight one. This approach first trains a strong teacher model, and then forces a more compact student model to imitate the teacher. Although the key to successful knowledge distillation is constructing a stronger teacher model, the teacher model using state-of-the-art NMT may remain inadequate owing to translation errors. Accordingly, using an inadequate teacher model severely degrades the student model due to error propagation, especially regarding words important to sentence meaning. To mitigate the degradation problem, we propose a knowledge distillation method using a lexical constraint as privileged information for NMT. The proposed method trains a teacher model with a lexical constraint, a list of words automatically extracted from a target sentence in the training data. We configure the lexical constraint according to the importance of words and the fallibility of NMT. Models trained with our proposed method result in improved translation compared with those trained with a baseline method for English$\leftrightarrow$German and English$\leftrightarrow$Japanese translation tasks under the condition without ensemble decoding and beam-search decoding.
}
\ekeywords{Neural Machine Translation, Knowledge Distillation, Lexical Constraints}

\headauthor{Mino et al.}
\headtitle{Lexically Constrained Knowledge Distillation for Neural Machine Translation}

\affilabel{Author_1}{}{NHK Science and Technology Research Laboratories}
\affilabel{Author_2}{}{Tokyo Institute of Technology}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}

Neural machine translation (NMT) \cite{Sutskever:14,Bahdanau:15,Vaswani:17} has significantly improved translation quality but is computationally expensive owing to using various techniques such as ensemble decoding, beam-search decoding, and very deep neural networks.
Using ensemble decoding and beam-search decoding results in decreased decoding speed, whereas deep neural networks require significant computational resources and time.
Efforts have been made to mitigate these issues.
A representative approach is knowledge distillation \cite{Kim:16}, which transfers the knowledge of a teacher model with a stronger learning capacity into an under-parameterized student model.
\citeA{Kim:16} proposed training a student model with training data where the source sentences were original but the target sentences were generated by the teacher model.
The key to the successful knowledge distillation is generating a stronger teacher model.
Therefore, various methods, such as using an ensemble of multiple NMT models \cite{Freitag:17} and online knowledge distillation from different checkpoints \cite{Wei:19}, have been developed.
Although state-of-the-art NMT models using these methods generate fluent sentences, some words may be mistranslated or under-translated, regardless of their importance.
Figure~\ref{sample} shows an example of under-translation using a teacher model and mistranslation using a student model in NMT, where the content of significant source words was translated incorrectly.
As just described, using an inadequate teacher model worsens the student model because of the accumulation and propagation of translation errors.
In this study, we assume that the appropriate teacher model needs to
 (1) enhance {\it the translation quality} of the generated target sentences from the training data and
 (2) have high {\it translation consistency} among the generated target sentences.
Translation quality refers to the performance regarding whether the generated target sentences are correctly translated from a source sentence, which is the basic criterion of knowledge distillation.
Translation consistency refers to the performance regarding whether words or phrases with the same meaning in the generated training data are unified into the same expressions.
This criterion is based on the assumption that a teacher model with a high translation consistency reduces the number of correct translation candidates and stabilizes the student model output.
In general, the NMT model output tends to be simpler and more standardized than the original training data.
This phenomenon is known as translationese \cite{Toral:19}.
Owing to the translationese effect, outputs translated with the teacher model exhibit high translation consistency.
However, the outputs seem to have low translation quality because of translation errors.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1
\begin{figure}[b]
\begin{center}
\includegraphics{29-4ia2f1.pdf}
\end{center}
\hangcaption{Translation examples with teacher and student models used for knowledge distillation in NMT from Japanese to English. Word (flow volume) was not translated by teacher model and mistranslated by student model in NMT output.}
\label{sample}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

To enhance the translation quality of the outputs translated with the teacher model while maintaining high translation consistency, we propose a lexically constrained knowledge distillation method inspired by a training technique using privileged information \cite{Lopez:16}.
\citeA{Lopez:16} proposed a method that improves a teacher model by using information that is available in the training phase but not in the inference phase.
We extended the idea of \citeA{Lopez:16} to NMT and trained the teacher model with a list of target words, called a lexical constraint, automatically extracted from target sentences in the training data.
Because the teacher model tends to generate words present in the constraint, the teacher model would be robust to these translation errors by setting mistranslated and under-translated target words as a constraint. 
Using a lexical constraint as privileged information is non-trivial in the knowledge distillation of NMT.
Borrowing an idea from \citeA{Gchen:20}, we used a lexical-constraint-aware NMT model by integrating a target constraint and source sentence together.
In terms of translation quality and translation consistency, we used two constraint types extracted from the target sentence, namely, content-word and mistranslated-word constraints.
The content-word constraint focuses on words with more important meanings compared with others in a sentence, which is helpful for NMT \cite{KChen:20}.
The mistranslated-word constraint focuses on words mistranslated or under-translated with vanilla NMT compared to a target sentence.

We conducted experiments to evaluate the proposed method that involved student models of different parameter sizes without ensemble decoding and beam-search decoding to speed up the decoding process.
Experimental results on English$\leftrightarrow$German translation tasks in the IWSLT2017 dataset \cite{Cettolo:12} and WMT'19 dataset \cite{Barrault:19} and English$\leftrightarrow$Japanese translation tasks in ASPEC \cite{Nakazawa:16} indicate that our proposed method can outperform a baseline method in terms of bilingual evaluation under study (BLEU) scores \cite{Papineni:02} among the student models.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Sequence-Level Knowledge Distillation}

Sequence-level knowledge distillation is conducted in two steps in almost the same manner as in \citeA{Kim:16}: training a teacher model and then training a student model with the trained teacher model.

In the first step, the teacher model was trained using the original training data.
The negative log-likelihood loss of the teacher model $L_{\rm TM}$ can be calculated as follows:
\begin{equation}
\label{slkd1}
{L_{\rm TM}}  = -\sum_{i=1}^{M}\log q({\bm y}^{i}|{\bm x}^{i};{\bm {\theta_{\rm TM}}}),
\end{equation}
where $M$ is the number of original training pairs, $i$ is the index of each training sentence pair,
${\bm x}^{i}=(x_1^i,x_2^i,...x_K^i)$ is a source-side sequence, ${\bm y}^{i}=(y_1^i,y_2^i,...,y_J^i)$ is a target-side sequence,
$q(\cdot|\cdot)$ is the conditional probability of the teacher model, and ${\bm {\theta_{\rm TM}}}$ is the set of model parameters for the teacher model.
The equation for $q({\bm y}^{i}|{\bm x}^{i};{\bm \theta})$ can be formulated as
\begin{equation}
\label{slkd2}
q(\bm{y}^{i}|\bm{x}^{i};{\bm {\theta_{\rm TM}}}) = \prod_{j=1}^{J}q(y_{j}^{i}|\bm{x}^{i},\bm{y}_{<j}^{i};{\bm {\theta_{\rm TM}}}).
\end{equation}

In the second step, a student model is trained using the outputs from the teacher model.
The student model is trained with two loss functions:
loss using the target sequences $L_{\rm SM}$ and sequence-level knowledge distillation $L_{\rm KD}$, derived from the target-side sequences of the original training data and teacher-generated sequences.
The loss using the target-side sequences, namely, $L_{\rm SM}$ is calculated as
\begin{equation}
\label{slkd3}
{L_{\rm SM}} = -\sum_{i}\log p({\bm y}^{i}|{\bm x}^{i};{\bm {\theta_{\rm SM}}}),
\end{equation}
where $p(\cdot|\cdot)$ is the conditional probability of the student model and ${\bm {\theta_{\rm SM}}}$ is the set of model parameters for the student model.
Equation~\ref{slkd3} is the same as Equation~\ref{slkd1} except for the probability distributions $p$ and $q$.
To calculate the loss for sequence-level knowledge distillation, there are some options such as using output distributions of the teacher model \cite{Hinton:15} and using generated sentences (one-hot distributions) \cite{Kim:16}.
To reduce the computational cost, we used teacher-generated sentences.
The loss for sequence-level knowledge distillation, namely, $L_{\rm KD}$ is calculated as
\begin{equation}
\label{slkd5}
{L_{\rm KD}} = -\sum_{i}\log p({\bm {\hat y}}^{i}|{\bm x}^{i};{\bm {\theta_{\rm SM}}}).
\end{equation}
where $\bm{\hat{y}}^{i}=(\hat{y}_1^{i},\hat{y}_2^{i},...,\hat{y}_J^{i})$ is a translated output obtained with the teacher model.
By interpolating between $L_{\rm SM}$ and $L_{\rm KD}$ described in Equations \ref{slkd3} and \ref{slkd5}, the total loss $L$ of the student model is calculated as follows:
\begin{equation}
\label{slkd6}
{L} = (1-\alpha){L_{\rm SM}} + \alpha{L_{\rm KD}}
\end{equation}
where $\alpha$ is a coefficient between $[0,1]$ that controls the contribution of the two loss functions.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Lexically Constrained Knowledge Distillation}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Sequence-Level Knowledge Distillation with Lexical Constraints}

To enhance the translation quality of the teacher model for sequence-level knowledge distillation, our proposed method trains a teacher model using a lexical constraint as privileged information, as shown at the top of Figure~\ref{model}.
We trained a teacher and student models to mimic the teacher's actions at the sequence-level in the following two steps.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 2
\begin{figure}[b]
\begin{center}
\includegraphics{29-4ia2f2.pdf}
\end{center}
\hangcaption{Overview of NMT adopted in knowledge distillation using the proposed method. Top shows the teacher model and bottom shows the student model. ${\bm x=(x_1,x_2,...,x_K)}$, ${\bm c}=(c_1,c_2,...,c_N)$, and ${\bm t}=(t_1,t_2,...,t_J)$ show source-, constraint-, and predicted-sequences respectively. ``$|$'' is special token for delimiter.}
\label{model}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

First, we trained a teacher model with a lexical constraint using the conditional probability of the teacher model. 
The conditional probability for the teacher model is calculated as follows:
\begin{equation}
\label{softkd1}
q(\bm{y}^{i}|\bm{x}^{i},\bm{c}^{i};\bm{\theta_{\rm CTM}}) = \prod_{j=1}^{J} q(y_{j}^{i}|\bm{x}^{i},\bm{c}^{i},\bm{y}_{<j}^{i};\bm{\theta_{\rm CTM}}),
\end{equation}
where $i$ is the index of the training data, and
$\bm{c}^{i}=(c_{1}^{i},c_{2}^{i},...,c_{N}^{i})$ denotes the lexical constraint that should be present in $\bm{\hat y}^{i}$ ($N$ is the number of words in the constraint) and
$\bm{ \theta_{\rm CTM}}$ is the set of model parameters for the teacher model.
Using Equation~\ref{softkd1}, the loss of the teacher model $L_{\rm CTM}$ is calculated as follows:
\begin{equation}
\label{softkd2}
{L_{\rm CTM}} = -\sum_{i} \log q(\bm{y}^{i}|\bm{x}^{i},\bm{c}^{i};\bm{\theta_{\rm CTM}}). \quad
\end{equation}
Then we trained the student model, which involves loss using the original training data $L_{\rm CSM}$ and loss of knowledge distillation with the constraint $L_{\rm CKD}$.
The $L_{\rm CSM}$ is calculated using the original training data as follows:
\begin{equation}
\label{softkd3}
{L_{\rm CSM}} = -\sum_{i}\log p({\bm y}^{i}|{\bm x}^{i};{\bm {\theta_{\rm CSM}}}),
\end{equation}
where $p(\cdot|\cdot)$ is the conditional probability of the student model and $\bm {\theta_{\rm CSM}}$ is the set of model parameters for the student model.
Using the teacher-generated sentence similar to that started in Equation~\ref{slkd5}, the loss of knowledge distillation with the constraint is calculated as
\begin{equation}
\label{softkd4}
{L_{\rm CKD}} = - \sum_{i}\log p(\bm{\hat{y}}^{i}_{\rm const}|\bm{x}^{i};\bm{\theta_{\rm CSM}}), \quad
\end{equation}
where $\bm{\hat y}_{\rm const}$ is the teacher-generated sequence with a constraint.
The difference between $L_{\rm KD}$ and $L_{\rm CKD}$ described in Equations~\ref{slkd5} and \ref{softkd4}, respectively, is the teacher-generated sequences
$\bm{\hat{y}}$ and $\bm{\hat y}_{\rm const}$ respectively.
$\bm{\hat y}_{\rm const}$ value used in Equation~\ref{softkd4} can be closer to the target sequence than that in Equation~\ref{slkd5} owing to using the constraint.
Using $L_{\rm CSM}$, $L_{\rm KD}$, and $L_{\rm CKD}$, the total loss for training the student model $L_{\rm C}$ is calculated as follows:
\begin{equation}
\label{softkd5}
{L_{\rm C}} = (1-\alpha-\beta){L_{\rm CSM}} + \alpha{L_{\rm KD}} + \beta{L_{\rm CKD}},
\end{equation}
where $\alpha$ and $\beta$ are coefficients between $[0,1]$ and $\alpha + \beta \leq 1$.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Constraints}

The proposed lexically constrained knowledge distillation presented in the previous section requires the extraction of constraints to enhance the translation quality of the teacher model.
We have two methods to extract constrained words automatically from the target sentence focusing on content words and mistranslated words.
Distilled data were generated leveraging these two constraint types separately.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Content Word Constraint}

The simplest method of extracting a lexical constraint is to randomly sample words from the target sentence, as stated in \citeA{Gchen:20}.
However, this method does not consider the importance of words in the meaning of a sentence.
\citeA{KChen:20} reported that using the information on the importance of words to enhance the translation quality of content words is helpful for NMT.
Based on this idea, we extracted the candidates of the constrained words from the target sentence using content-word recognition based on term frequency-inverse document frequency (TF-IDF).
As a constraint, we determine the fixed number of words with high TF-IDF scores above the threshold in the target sentence.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Mistranslated Word Constraint}

The other method for extracting a constraint is based on whether the predicted words are mistranslated or dropped by NMT.
Adding a mistranslated-word constraint would be incorporated into the model to correctly translate not only the words themselves but also their neighboring words.
We extract the mistranslated words as a constraint using the following procedure:
First, given parallel training data, we train an NMT model and translated the source sentences using this model.
Then, we define the mistranslated words as those that do not appear in the translated sentence but are contained in the target sentence.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[b]
\input{02table01.tex}
\hangcaption{Details of each dataset. ``sent.'' is number of sentences, and ``tokens'' and ``types'' is number of tokens and types of left- and right-side languages in each language pair.}
\label{corpus}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experiments}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Datasets}

We conducted experiments using three parallel corpora with different data sizes and language pairs.
\begin{itemize}
\item {\bf TED Talk Corpus (English$\leftrightarrow$German).}
We conducted an experiment on the IWSLT 2017 \cite{Cettolo:12} dataset based on the TED Talks.
We used the ``train'' set for training and the ``dev2010'' set for validation for all the tasks.
The ``tst2015'' was used for testing the English$\leftrightarrow$German tasks.
\item {\bf WMT'19 (English$\leftrightarrow$German).}
We conducted another experiment using the WMT'19 dataset \cite{Barrault:19}, which is a large corpus.
We used the ``news2018'' set for validation for all tasks.
The ``news2019'' set was used for testing the English$\leftrightarrow$German tasks.
\item {\bf ASPEC (Japanese$\leftrightarrow$English).}
We conducted another experiment using a Japanese-English paper abstract corpus called ASPEC \cite{Nakazawa:16}.
The ASPEC dataset contains 3M bilingual pairs in the training data with the automatically estimated similarity scores \cite{Utiyama:07} provided for each pair.
We used only the first 2M pairs with high similarity scores to train the models.
\end{itemize}
Table~\ref{corpus} lists the statistics for each corpus.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{System}

Our method can be implemented using a vanilla NMT system with encoder and decoder modules.
In this experiment, we concatenated a source sentence and constrained words with a special token (namely ${|}$) to be input into the encoder for the teacher model, as illustrated in Figure~\ref{model}.
The teacher model in this system does not ensure that all constrained words appear in the output, namely uses soft constraints.
As a side benefit, translation quality is maintained when using soft constraints compared to when using hard constraints, as \citeA{Gchen:20} reported.
For both content word and mistranslated word constraints, we used at most five words from candidates sorted based on their TF-IDF scores \cite{KChen:20}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Settings}

We used the Moses toolkit\footnote{\url{https://github.com/moses-smt/mosesdecoder}} to clean and tokenize English and German sentences and KyTea \cite{Neubig:11} to tokenize Japanese sentences.
Then, we used a vocabulary of 48K tokens based on joint byte-pair encoding (BPE) \cite{Sennrich:16B} for the source and target.
The frequency threshold for the vocabulary filter was set to 35.

For the NMT model, we used the encoder and decoder of the transformer model \cite{Vaswani:17}.
The transformer model contains a multi-headed attention mechanism, applied as self-attention, and a position-wise fully connected feed-forward network.
The encoder converts a source sentence into a sequence of continuous representations, while the decoder generates a target sentence.
For training the teacher models, we used a 12-layer encoder and decoder respectively.
For training the student models, we experimented with different sized encoder and decoder layers called as ``large'' for 12-12 layers, ``small'' for 3-3 layers, and ``tiny'' for 1-1 layers.
This system was implemented using the Sockeye 2 toolkit \cite{Hieber:20}.
All models were trained within at most three days on Nvidia A100 Tesla GPUs with 40-GBs memory.

When training the teacher and student models, we applied stochastic gradient descent with Adam \cite{Kingma:15} optimizer with a learning rate of 0.0002 multiplied by 0.7 after every eight checkpoints.
We set the batch size to 3000 tokens and the maximum sentence length to 150 BPE tokens owing to memory limitations.
The checkpoint interval was set to 4,000 batches (updates).
We applied early stopping, which stopped the training if the validation loss did not improve over 32 checkpoints.
The dropout rates for the attention, feed-forward, and embedding layers were set as reported in Table~\ref{dropout}.
The values of dropout rates were determined after testing with $0.01$, $0.05$, $0.1$, $0.2$, $0.3$, and $0.5$ using development data.
The hyperparameters $\alpha$ and $\beta$ stated in Equation~\ref{softkd5} were set to $0.33$ and $0.33$, respectively, to be implemented equally between the three types of losses.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 2
\begin{table}[t]
\input{02table02.tex}
\caption{Values of dropout rates to train teacher and student models for each corpus.}
\label{dropout}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

When translating with the teacher models, we used a beam search with a beam width of 10 and an ensemble of five models with different seeds.
On the other hand, when decoding with student models, we did not use ensemble decoding or beam-search decoding to reduce computational costs in the decoding process.
We also used an average of the model parameters over the training steps, as reported in \citeA{Wei:19}.

In both of the training and decoding phases, we used a label smoothing\footnote{We analyzed the effect of the label smoothing in Section~\ref{sec:analysis}.} of $0.1$ after testing with $0.0$ and $0.1$.

As the other model hyperparameters, we used the default Sockeye~2 parameters.

To evaluate the translation quality, we calculated case-insensitive BLEU \cite{Papineni:02} scores by using SacreBLEU\footnote{\url{https://github.com/mjpost/sacrebleu}} \cite{PostA:18}.
We trained five student models with different seeds and used the median BLEU score of the five translation results without ensemble decoding and beam-search decoding.
We experimented with our method using both content-word and mistranslated-word constraints (Cont. \& Mistrans.-KD).
For baseline systems, we used knowledge distillation models without any constraints (Baseline-KD).
For the significance test, we used a paired-bootstrap.py\footnote{\url{https://github.com/neubig/util-scripts/blob/master/paired-bootstrap.py}} with a sample size of $10,000$.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Main Results}
\label{main-results}

Table~\ref{bleu} lists the evaluation results of the outputs obtained from the baseline and student models for knowledge distillation on English$\leftrightarrow$German tasks with the TED Talk Corpus and WMT'19 and English$\leftrightarrow$Japanese tasks with ASPEC.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[t]
\input{02table03.tex}
\hangcaption{BLEU scores of each student model on English$\leftrightarrow$German and English$\leftrightarrow$Japanese translation results compared in terms of size of encoder and decoder layers (large:12-12, small:3-3, and tiny:1-1) using knowledge distillation methods. We did not use ensemble or beam-search decoding when translating with each student model. ``Baseline-KD'' denotes the baseline method with knowledge distillation that does not use any constraints. ``Cont. \& Mistrans.-KD'' denotes our proposed knowledge distillation method using the content-word and the mistranslated-word constraints. $\Delta$: BLEU score improvements with our methods over the ``Baseline-KD''. $\dag$ indicates significant differences compared to the ``Baseline-KD'' with a significance level of $0.05$.}
\label{bleu}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Regarding the English$\leftrightarrow$German translation results with the TED Talk Corpus, our proposed method improved in translation performance compared with the baseline (Baseline-KD) for all layer sizes.
In particular, the improvements of our proposed method with a ``tiny'' sized layers (1-1) were more prominent.

Regarding the English$\leftrightarrow$German translation results with WMT'19, although our proposed method improved the BLEU scores for ``small'' and ``tiny'' sizes of layers by $+0.4$ to $+0.9$, respectively, it only yielded a slight improvement of $+0.1$ for ``large'' sized layers (12-12).
Compared to the TED Talk Corpus, WMT'19 has significantly more data and includes various domains of data.
It appears that the effectiveness of our proposed method is affected by the amount of data or the domain of data.

Regarding the English$\leftrightarrow$Japanese translation results with ASPEC, the same trend as those depicted in the results using the TED Talk Corpus is observed.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Analysis}
\label{sec:analysis}

We analyzed constraint extraction, comparison between models 
\pagebreak
with and without knowledge distillation, effect of using ensemble decoding and beam-search decoding for student model, training data generated with each teacher model, decoding speed, and effect of label smoothing.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Constraint Extraction}

To verify the effectiveness of our proposed method in extracting constraints, namely, content-word and mistranslated-word constraints, we compared among the versions of our proposed method reported in Table~\ref{ana3}, namely, the content word constraint (Const.-KD), the mistranslated word constraint (Mistrans.-KD) and both constraints (Const. \& Mistrans.-KD), on English$\leftrightarrow$
\linebreak
German translation tasks using the TED Talk Corpus.
Furthermore, we compared the proposed method with a method that randomly extracts constrained words from a target sentence (Rand.-KD).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 4
\begin{table}[b]
\input{02table04.tex}
\hangcaption{BLEU scores on English$\leftrightarrow$German translation results of TED Talk Corpus using each student model under the condition without ensemble decoding and beam-search decoding.
Constraints of each method are the content word constraint (Const.-KD), the mistranslated word constraint (Mistrans.-KD) and both constraints (Const. \& Mistrans.-KD).
Furthermore, we experimented the method of extracting constrained words from a target sentence randomly (Rand.-KD).
}
\label{ana3}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The results reported in Table~\ref{ana3} indicated that ``Rand.-KD'' performed better than ``Baseline-KD'', except for German$\rightarrow$English tasks with ``large'' sized layers, but worse than our proposed methods.
A comparison among the versions of our proposed method, namely, the content word constraint (Const.-KD), the mistranslated word constraint (Mistrans.-KD) and both constraints (Const. \& Mistrans.-KD), Const. \& Mistrans.-KD achieved the best results for all layer sizes.
These results indicate that both constraints are significant for the proposed method.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 5
\begin{table}[b]
\input{02table05.tex}
\hangcaption{BLEU scores regarding translation results using models without knowledge distillation and proposed knowledge distillation models for each layer size (``large'' for 12-12 layers, ``small'' for 3-3 layers, and ``tiny'' for 1-1 layers). ``Large'' models using ensemble decoding and a beam-search decoding with a beam width of 10 (bw=10) listed in the fourth column are used to train each student model of ``Baseline-KD''. The eighth column depicts the results obtained with proposed method (Cont. \& Mistrans.-KD). $\Delta_{1}$: $\text{(eighth column)}-\text{(seventh column)}$. $\Delta_{2}$: $\text{(eighth column)}-\text{(fourth column)}$.}
\label{ana-teacher}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Comparison between Models with and without Knowledge Distillation}

Table~\ref{ana-teacher} lists experimental results of models with and without knowledge distillation for each layer size.
For models without knowledge distillation, results when using an ensemble decoding and a beam-search decoding are also shown in columns 4 to 7.
The models constructed with ``large'' sized layers using ensemble decoding and beam-search decoding with a beam width of 10 (bw=10) listed in the fourth column were used for teacher models of ``Baseline-KD''.
The values of the $\Delta_{1}$ indicate the differences between models with and without knowledge distillation when not using ensemble decoding and beam-search decoding $\text{(eighth column)} - \text{(seventh column)}$.
The values of the $\Delta_{2}$ indicate the differences between the model with knowledge distillation not using ensemble decoding and beam-search decoding and that without knowledge distillation using ensemble decoding and beam-search decoding $\text{(eighth column)} - \text{(fourth column)}$.

From the values of the $\Delta_{1}$, we can find that the proposed model with knowledge distillation totally achieved higher BLEU score than the model without knowledge distillation.
These results underline the effectiveness of the knowledge distillation method.
Furthermore, from the values of the $\Delta_{2}$, the proposed model with ``tiny'' sized layers without ensemble decoding and beam-search decoding achieved a higher BLEU score than that without knowledge distillation of ``tiny'' sized layers using both decoding techniques in all tasks.
These results suggest that the smaller the layer size adopted in a model, the greater the effect of the knowledge distillation methods.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Effect of Using Ensemble Decoding and Beam-Search Decoding for the Student Model}

Although our work aimed to improve translation quality under the condition of not using ensemble decoding and beam-search decoding during translation to speed up the decoding process, we conducted experiments involving the student models with large layer sizes (12-12) using ensemble decoding and beam-search decoding for analysis.
Moreover, we evaluated the teacher model of the Baseline-KD.
Table~\ref{ana2} lists the experimental results of this analysis.
Notably, the eighth column (no ensemble, bw=1) in Table~\ref{ana2} is the same as the fourth column (size of layers, large) in Table~\ref{bleu}.

We assumed that the ``teacher'' model using ensemble decoding and beam-search decoding with a beam width of 10 (bw=10) performed the best because this model was trained with only the genuine training data.
However, there were no or small difference between ``teacher'' and ``student'' with ``Const. \& Mistrans.-KD'' method for English$\leftrightarrow$German and Japanese$\rightarrow$English tasks even when compared with models without ensemble decoding and beam-search decoding.
For example, in the English$\rightarrow$German task using the TED Talk Corpus, the student model without ensemble decoding and beam-search decoding (no ensemble and bw=1) achieved a higher BLEU score ($29.5$) compared to that of the teacher model using ensemble decoding and beam-search decoding with beam width of 10 ($29.4$).
Another evaluation based on $\Delta$ in Table~\ref{ana2} can be made.
The values in the $\Delta$ column underlines the difference between constructing models with and without both of the decoding techniques.
The $\Delta$ values of the models using knowledge distillation were lower than those of the baseline models without knowledge distillation.
These results show that the translation quality of the methods with knowledge distillation degrades less compared with methods without knowledge distillation when not using ensemble decoding or beam-search decoding.
For beam-search decoding, the same observation was reported by \citeA{Kim:16}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 6
\begin{table}[t]
\input{02table06.tex}
\hangcaption{BLEU scores based on translation results using each student model with the ``large'' sized layers (12-12) for comparison in terms of using an ensemble decoding (ensemble with five different models and no ensemble) and a beam-search decoding (with a beam width of 10 and 1 (bw=10 and bw=1)). In contrast to the results reported in Table~\ref{bleu}, ensemble decoding and beam-search decoding were used for training each student model. Five ensemble models with different seeds were used. $\Delta$: (BLEU score of ensemble model with beam width of 10 $\text{(fifth column))} - \text{(BLEU}$ score of no-ensemble model with beam width of 1 (eighth column)). ``Teacher'' with ensemble and bw=10 is teacher model to train a student model of ``Baseline-KD''. The eighth column (no ensemble, bw=1) in Table~\ref{ana2} is same as the fourth column in Table~\ref{bleu}.}
\label{ana2}
\vspace{-0.5\Cvs}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Training Data Generated with Each Teacher Model}

We analyzed the training data generated with a teacher model focusing on two criteria: 
\pagebreak
translation quality and translation consistency.
Table~\ref{ana1} reports the detailed information regarding teacher-generated sentences with each teacher model.
Translation quality refers to the performance, that is, whether the generated target sentences are correctly translated from a source sentence.
Translation consistency refers to the performance regarding whether words or phrases with the same meaning in the generated training data are unified into the same lexicons.
Because it is difficult to evaluate translation quality and translation consistency correctly, the superficial similarity between the generated target sentence and the reference was used as an approximation of translation quality; additionally, the number of word types in the training data was used as an approximation of translation consistency.
We assumed that data with high translation quality tend to have a high degree of similarity between the generated sentence and the reference, whereas data with high translation consistency among the generated target sentences tend to have a small number of word types.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 7
\begin{table}[p]
\input{02table07.tex}
\hangcaption{Details regarding target sentences translated with each teacher model of knowledge distillation. When translating with a teacher model to produce target sentences for a training data, we used ensemble decoding (ensemble with five different models) and beam-search decoding (with a beam width of 10). Different random seeds were set according to the five models. ``Word types'' denotes the number of token types for the target language before applying BPE. Matching rate indicates the ratio of the number of words appearing in the given output words of content-word and mistranslated-word constraints over training data. The ``Similarity'' scores refer to BLEU scores calculated using all target sentences of the training data as reference translations. Reference shows the original training data.}
\label{ana1}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

In terms of the criterion of translation quality, we analyzed the matching rates in the fifth and sixth columns, which refer to the ratio of the number of words appearing in the output of the given words of each constraint over the training data.
The matching rates for the proposed method (Cont.~\&~Mistrans.-KD) were higher than those of the baseline (Baseline-KD).
Cont.-KD achieved the highest scores in the ``Cont.'' matching rates and Mistrans.-KD achieved the highest scores in the ``Mistrans.'' matching rates.
Moreover, we analyzed the criterion of translation quality criterion based on the similarity scores in the seventh column.
The similarity scores refer to the BLEU scores calculated using all target sentences of the training data as references.
Regarding the comparison between the proposed method (Cont. \& Mistrans.-KD) and the baseline method (Baseline-KD), observe that the proposed method achieved higher similarity scores.
Regarding the comparison between the proposed method and the other knowledge distillation methods (Rand.-KD, Cont.-KD, Mistrans.-KD), the trend in the results of the similarity score changed depending on the task.
These results appear that the Cont. and Mistrans. matching rates are more significant factors for the knowledge distillation than the similarity.
In terms of translation consistency, we analyzed the word types in the fourth column.
The word types in the knowledge distillation methods were lower than those in the reference.
These results suggest that the outputs of teacher models using knowledge distillation methods attain high translation consistency.
The baseline method obtained the highest translation consistency among the knowledge distillation methods, but lower translation quality than the proposed method.

From these analyses, it appears that the proposed method succeeds in obtaining a high translation quality we aim for and higher translation consistency than that of the reference.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Decoding Speed}

We measured the average number of source words translated per second to assess the decoding.
From the results reported in Table~\ref{ana4}, the model with a ``tiny'' sized layers without ensemble decoding and beam-search decoding (no ensemble and bw=1) was approximately forty times faster than the model with ``large'' sized layers with ensemble decoding of five models and beam-search decoding with a beam width of 10 (ensemble and bw=10).
Although ensemble decoding, beam search decoding, and an increasing number of layers greatly decreased decoding speed, they had little effect on translation quality in some cases.
The values reported in parentheses in Table~\ref{ana4} indicate the BLEU scores for each model.
For example, the model with ``small'' sized layers using ensemble decoding and no beam-search decoding (ensemble and bw=1) attains almost no improvement in translation quality compared to the model with ``small'' sized layers using no ensemble decoding and beam-search decoding with a beam width of 1 (no ensemble and bw=1).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 8
\begin{table}[b]
\input{02table08.tex}
\hangcaption{Average number of source sentences translated per second with English$\rightarrow$German models on single Nvidia A100 Tesla GPU with ensemble decoding (ensemble with five different models and no ensemble) and beam width (bw=1 and bw=10) for each layer size (``large'' for 12-12 layers, ``small'' for 3-3 layers, and ``tiny'' for 1-1 layers). The values in parentheses indicate the BLEU scores for each model.}
\label{ana4}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Effect of Label Smoothing}

In both of the training and decoding phases of the abovementioned experiments, we used a label smoothing of $0.1$ after testing with $0.0$ and $0.1$ as reported in Table~\ref{ana-ls}.
Although \citeA{Muller:19} reported that if a teacher model is trained with label smoothing, knowledge distillation into a student model is much less effective, we could not observe the same trend in our results.
One of the differences between our method and theirs is that the student model of our method is trained to mimic the output token of the teacher model, whereas the student model of \citeA{Muller:19} is trained to mimic the output distribution of the teacher model.
Because the label smoothing technique affects the output distribution of the model, the method of imitating the output distribution would be more affected by label smoothing than the method of imitating one-hot output tokens.
Therefore, our proposed method prevented the deterioration in translation quality.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 9
\begin{table}[t]
\input{02table09.tex}
\hangcaption{BLEU scores of each student model when using label smoothing or not on TED Talk Corpus and ASPEC. ``Baseline-KD'' denotes baseline method with knowledge distillation. ``Cont. \& Mistrans.-KD'' denotes our proposed knowledge distillation method using the content-word and mistranslated-word constraints.}
\label{ana-ls}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Work}

Knowledge distillation, proposed by \citeA{Bucila:06} and popularized by \citeA{Hinton:15}, compresses the knowledge of a large and computationally expensive teacher model into a compact student model.
The concept of knowledge distillation involves the training of a student model on the transfer training data provided by a strong teacher model.
Knowledge distillation has been widely adopted in various learning tasks including machine translation \cite{Kim:16,Freitag:17,Wei:19,Wang:21}.
\citeA{Kim:16} proposed two extensions of knowledge distillation to machine translation tasks: word-level and sequence-level knowledge distillation methods.
\citeA{Kim:16} also proposed interpolated knowledge distillation for the sequence-level knowledge distillation method using teacher-generated data and the original training data.
\citeA{Freitag:17} proposed an ensemble of multiple NMT models for training a strong teacher model.
\citeA{Wei:19} proposed an online distillation and an exponential moving average method, in which a student model learns from different checkpoints for only the word-level knowledge distillation.
\citeA{Wang:21} proposed another strategy for knowledge distillation that selects suitable samples from the entire training dataset.
This approach was implemented using the data provided by a strong teacher model.
Because our proposed method focuses on generating a strong teacher model, this approach is beyond the scope of this study.
\citeA{Zhou:20} reported that knowledge distillation could reduce the complexity of datasets for a non-autoregressive model.
Moreover, they observed a strong correlation between the capacity of the model and the complexity of the distilled data that provides the best translation quality.
\citeA{Lopez:16} unified knowledge distillation and privileged information \cite{Vapnik:15}, which can only be used during training.
The proposed method was inspired by the method proposed by \citeA{Lopez:16}, which was mainly designed for classification tasks.

Lexically constrained NMT has been developed for domain-specific machine translation and interactive machine translation to ensure that specific translated words are present in the outputs.
\citeA{Hokamp:17} proposed a grid beam-search-decoding algorithm to ensure that all lexical constraints appear at the translation output.
Related methods have been proposed to improve the beam-search algorithm \cite{Post:18,Hu:19}.
These methods have a higher computational cost than conventional beam search.
Another direction of lexically constrained NMT modifies the training process \cite{Gchen:20,KChen:20}.
These methods do not guarantee the presence of the target-side constraints in the translation outputs.
\citeA{Gchen:20} proposed constructing a code-switching corpus by appending target tokens for a constraint after the source sentence without the bilingual dictionary.
Their proposed model learns to use the constraint automatically without explicit alignment information and improve translation quality.
\citeA{KChen:20} proposed content-word-aware NMT by exploiting word frequency information to distinguish content and function words and explored the importance of words for NMT.
We applied a soft lexically constrained NMT to knowledge distillation.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}

We addressed the problem that the current NMT approaches with knowledge distillation did not translate correctly regarding the importance of word.
We proposed a knowledge distillation method with a lexical constraint for NMT.
Our method improved the translation quality of a student model without ensemble decoding and beam-search decoding in terms of BLEU scores in English$\leftrightarrow$German and English$\leftrightarrow$Japanese translation tasks.
Furthermore, we observed small improvements in the student models when using not only beam-search decoding but also ensemble decoding compared to models without knowledge distillation.
In future research, we plan to tune the hyperparameters of the proposed method and analyze the robustness of the proposed method.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment

These research results were obtained from the commissioned research (No. 225) by National Institute of Information and Communications Technology (NICT), Japan.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.6}
\bibliography{02refs}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Hideya Mino}{received his B.S., M.S., and Ph.D. degrees in computer science from the Tokyo Institute of Technology in 2002, 2004 and 2022, respectively. He joined NHK (Japan Broadcasting Corporation) in 2004 and has worked for NHK Science and Technology Research Laboratories since 2009. He was a Visiting Researcher at National Institute of Information and Communications Technology from 2013 to 2017. His research interests include natural language processing.}

\bioauthor[:]{Kazutaka Kinugawa}{received his B.S. and M.S. degrees from the University of Tokyo in 2016 and 2018, respectively. He joined NHK (Japan Broadcasting Corporation) in 2018 and has worked for NHK Science and Technology Research Laboratories since 2020. His research interests include natural language processing.}

\bioauthor[:]{Hitoshi Ito}{received his B.S. degree in electrical and electronic engineering from Kobe University in 2010 and his M.S. degree from Kyoto University in 2012, respectively. He joined NHK (Japan Broadcasting Corporation) in 2012 and has worked for NHK Science and Technology Research Laboratories since 2015. His research interests include natural language processing.}

\bioauthor[:]{Isao Goto}{received his B.S. and M.S. degrees from Waseda University in 1995 and 1997, respectively, and received his Ph.D. degree from Kyoto University in 2014. He joined NHK (Japan Broadcasting Corporation) in 1997 and has worked for NHK Science and Technology Research Laboratories since 1999. He was a Visiting Researcher at Advanced Telecommunications
Research Institute International from 2004 to 2006 and a Visiting Researcher at National Institute of Information and Communications Technology from 2008 to 2013. His research interests include machine translation.}

\bioauthor[:]{Ichiro Yamada}{received his B.S., M.S., and Ph.D. degrees from Nagoya University in 1991, 1993 and 2009, respectively. He joined NHK (Japan Broadcasting Corporation) in 1993 and has worked for NHK Science and Technology Research Laboratories since 1996. He was a Visiting Scholar at Stanford University from 2003 to 2004 and a Visiting Researcher at National Institute of Information and Communications Technology from 2008 to 2011. His research interests include machine translation and knowledge acquisition.}

\bioauthor[:]{Takenobu Tokunaga}{received his B.S., M.S., and Ph.D. degrees from Tokyo Institute of Technology in 1983, 1985 and 1991, respectively. He is currently a professor at School of Computing, Tokyo Institute of Technology. His current interests include natural language processing, in particular, building and managing language resources, discourse analysis, dialogue systems and applications of language technologies to intelligent information access and education.}


\end{biography}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\biodate




\end{document}

