\documentclass[english]{jnlp_3.7}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{hangcaption_jnlp}
\usepackage{array}
\allowdisplaybreaks


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition

\usepackage{booktabs}
\usepackage{bm}
\usepackage{multirow}

\newcommand{\yn}[1]{\textcolor{black}{#1}}
\newcommand{\ynn}[1]{\textcolor{black}{#1}}
\usepackage{color}
\newcommand{\rev}[1]{\textcolor{black}{#1}}
\newcommand{\revv}[1]{\textcolor{black}{#1}}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{31}
\Number{2}
\Month{June}
\Year{2024}
%%%%\Issuetitle{「　　　　」特集号}

\received{2023}{11}{1}
\revised{2024}{2}{8}
\accepted{2024}{3}{4}

\setcounter{page}{534}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Revisiting Pre-training of Embedding Layers \\ in Transformer-based Neural Machine Translation}
\eauthor{Masato Neishi\affiref{Author_1} \and Naoki Yoshinaga\affiref{Author_2}} 

\eabstract{%
Recent trends in the pre-training and fine-tuning paradigm have made significant advances in several natural language processing tasks, including machine translation (MT), particularly for low-resource situations.
However, it is reported that leveraging out-of-domain data is not as effective, or sometimes even harmful, in MT tasks in high-resource situations, where further improvement is still needed.
In this study, we focus on domain-specific dedicated neural machine translation (NMT) models, which still have the advantage in a high-resource situation as concerns translation quality and inference cost. 
We revisit the in-domain pre-training of embedding layers in Transformer-based NMT models, in which the embeddings are pre-trained with the same training data as the target translation task, considering the large impact of the domain discrepancy between the pre-training and fine-tuning (or training) in MT tasks.
Experiments on two translation tasks, ASPEC English-to-Japanese and WMT2017 English-to-German, demonstrate that the in-domain pre-training of embedding layers in a Transformer-based NMT model provides performance improvement without any negative impact and contributes to earlier convergence in training.
Through additional experiments, we confirmed that pre-training of the embedding layer of the encoder is more important than that of the embedding layer of the decoder, and the impact does not vanish as the training data size is increased.
An analysis of the embeddings revealed the large impact of the pre-training of the embedding layers on the low-frequency tokens.%
}

\ekeywords{Machine Translation, Pre-training of Embedding Layer, Pre-trained Embedding, Transformer}

\headauthor{Neishi and Yoshinaga}
\headtitle{Revisiting Pre-trained Embeddings in Transformer-based NMT}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\affilabel{Author_1}{}{Graduate School of Information Science and Technology, The University of Tokyo (currently at Mirai Translate, Inc.)}
\affilabel{Author_2}{}{Institute of Industrial Science, The University of Tokyo}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\renewcommand{\thefootnote}{\fnsymbol{footnote}}
\footnote[0]{\yn{The embedding-layer initialization studied in this article was originally presented at the 4th Workshop on Asian Translation (WAT2017) for classical recurrent neural network (RNN)-based NMT model, and was evaluated only on a single language pair and translation direction (English to Japanese) \cite{neishi2017bag}. This article comprehensively re-evaluates this method in Transformer-based NMT for two language pairs, and further analyzes the embeddings obtained with and without our initialization method to clarify the model- and language-agnostic advantages of the method.}}
\renewcommand{\thefootnote}{\arabic{footnote}}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S1
\section{Introduction}\label{sec:introduction}
Significant advances in deep learning have realized various natural language processing (NLP) applications for practical use, but their reliability is still not high; hence, a need for improvements still exists in several cases.
The performance of machine translation (MT) systems in high-resource language pairs and domains seems to have reached saturation.
Consequently, the focus of studies has recently shifted to low-resource situations, as represented by multi-lingual MT models \cite{tiedemann-thottingal-2020-opus,liu-etal-2020-multilingual-denoising,fan2020englishcentric,nllbteam2022language}.

To mitigate the data sparseness problem, the pre-training and fine-tuning paradigm has become a standard approach in several NLP tasks and has exhibited remarkable improvements, particularly in low-resource settings.
However, in MT tasks in high-resource situations, some researchers have reported that leveraging out-of-domain data is not as effective or sometimes even harmful, for instance, in multi-domain settings where the data sizes of the domains are imbalanced or the domains are distant \cite{koehn-knowles-2017-six,chu-etal-2017-empirical,jiang-etal-2020-multi-domain,morishita-etal-2020-jparacrawl}.
In addition, a cost problem with models based on this paradigm exists because they are often excessively large in size.
Recent studies on large language models (LLMs) have shown the good translation ability of LLMs, which rivals that of the dedicated neural machine translation (NMT) models \cite{NEURIPS2020_1457c0d6,JMLR:v24:22-1144,vilar-etal-2023-prompting}, while the size of them is far larger.

\rev{In this study, we focus on domain-specific dedicated NMT models and revisit the in-domain pre-training of embedding layers that use only the provided translation training data to improve the NMT model.
Dedicated NMT models have not lost their advantage in high-resource settings as concerns both translation quality and inference cost compared to recent pre-trained and fine-tuned models.
We expect that better utilizing a given domain-specific dataset would be more effective than using additional out-of-domain datasets.}

Previous studies have shown that the pre-trained embeddings help find a better optimum and result in better model performance and earlier convergence in recurrent neural network (RNN)-based NMT models \cite{neishi2017bag,abdou-etal-2017-variable,qi-etal-2018-pre,zhang-et-al-2020}.
Is this also effective for a better-performing Transformer-based NMT model \cite{vaswani2017attention}?
To answer this question, we conducted a comprehensive study on the pre-trained embeddings for a Transformer-based NMT model on two MT tasks: ASPEC English-to-Japanese \cite{nakazawa16} and WMT2017 English-to-German \cite{bojar-etal-2017-findings}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
With experiments and analysis, we confirmed the following results:
\begin{itemize}
\item Among Continuous Bag-of-Words (CBoW), skip-gram, subword information skip-gram (SISG), and Global Vectors (GloVe), skip-gram is slightly better for pre-training the embedding layers of a Transformer-based NMT model than the others, while none of them have a negative impact (\S \ref{sec:comp_methods}).
\item Even with a large learning rate, pre-trained embeddings are effective provided the model is stably trained, and they also contribute to earlier convergence (\S \ref{comp_learning}).
\item Among the embedding layers in the model, that in the encoder is more important than that in the decoder for the pre-training (\S \ref{comp_enc_dec}).
\item While Pre-trained embeddings are particularly effective when the training data size is small but not excessively small, they retain the impact also in high-resource situations (\S \ref{comp_datasize}).
\item The pre-training has a large impact on the embeddings of low-frequency tokens, preventing the hubness problem of the embeddings (\S \ref{sec:analysis})\yn{, and is, therefore, more effective when a larger vocabulary size is used (\S \ref{sec:vocab}).}
\item \yn{Use of additional in-domain monolingual data in the pre-training boosts its advantage (\S \ref{sec:add_mono}).}
\item \yn{Use of the warm-up strategy in training the Transformer-based NMT models does not affect the impact of the pre-training (\S \ref{sec:warm}).}
\end{itemize}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2
\section{Related Work}\label{sec:relatedwork}
In the early stages of the neural networks (NNs) in NLP, pre-trained word embeddings were widely used as inputs for machine learning models in several NLP tasks \cite{turian-etal-2010-word,iyyer-etal-2015-deep,arora2017a,shen-etal-2018-baseline}, in which the embeddings were frozen.
As the NN models have deepened, training models from scratch has become more standard, and some researchers have explored the use of pre-trained embeddings to initialize the embedding layers of NN models \cite{kim-2014-convolutional,wieting2016universal,lample-etal-2016-neural,ma-hovy-2016-end,neishi2017bag,abdou-etal-2017-variable}.
After pre-trained language models (LMs) such as ELMo \cite{peters-etal-2018-deep} and BERT \cite{devlin-etal-2019-bert} were introduced, which were also intended to produce deep contextualized word embeddings, the pre-training and fine-tuning paradigm became the standard approach, and this trend has been continuing even until the recent trend of LLMs.

The previous approach of initializing embedding layers with pre-trained word embeddings can also be considered as a pre-training and fine-tuning paradigm; the differences are the model size, pre-training data size, and target parameters for initialization.
In the use of pre-trained word embeddings as the initializer, the target of the pre-training is limited to the embedding layers of the model, and the other parameters are randomly initialized and trained later.
Some researchers have explored partially pre-trained models that target more parameters than the embedding layers \cite{ramachandran-etal-2017-unsupervised,imamura-sumita-2019-recycling,clinchant-etal-2019-use,Zhu2020Incorporating} before pre-trained LMs, such as BART \cite{lewis-etal-2020-bart} in which pre-training covers all the parameters of the resulting model. 
\yn{The differences in the pre-training areas of these pre-training approaches are shown in Figure \ref{fig:pretraining_area}.}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F1
\begin{figure*}[t]
\begin{center}
\includegraphics{31-2ia9f2.pdf}
\end{center}
\hangcaption{\ynn{Target areas of the pre-training: only embedding layers (left, this study), partial (middle; \textit{e.g.}, \protect\cite{ramachandran-etal-2017-unsupervised}), and full (right; \textit{e.g.}, \protect\cite{lewis-etal-2020-bart})}}
\label{fig:pretraining_area}
\end{figure*}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



Parameter initialization methods are the most relevant to this study, and some well-known methods exist, such as Xavier initialization \cite{pmlr-v9-glorot10a} and He initialization \cite{he2015delving}, which lead to a better local optimum.
Initialization methods can contribute to stabilizing the training of very deep Transformer models \cite{zhang-etal-2019-improving,Liu2020VeryDT}.
Concerning the stabilization of the training, bypass connections \cite{7780459,wang2022deepnet,takase-etal-2023-b2t} are often used to help the back-propagation of the training loss through the network preventing the vanishing gradient problem.
With the introduction of the Transformer model, a warm-up strategy was employed \cite{vaswani2017attention}, which directly controls the learning rate of the optimizer to stabilize the training in the early stage.
In addition to the warm-up strategy, recent NN models employ learning rate schedulers to control the learning rate over the entire training period.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3
\section{Pre-training of Embedding Layers in NMT}\label{sec:approach}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.1
\subsection{Why Pre-train Embedding Layers}
Pre-trained embeddings were originally used to mitigate the data sparseness problem by leveraging the large amounts of monolingual data for the pre-training.
In this study, the main motivation for the pre-training of the embedding layers is the flow of back-propagation, which propagates losses from the output layer to the input layer.
As the distance from the location where the training loss is calculated increases, the gradient propagating through the model decreases or vanishes.
To build deeper NN models, bypass connections \cite{7780459,wang2022deepnet,takase-etal-2023-b2t} are often employed to stabilize loss propagation.
Pre-training of the embedding layers is another approach to the gradient problem, which directly pre-trains the parameters of the shallow layers of NNs.
As concerns the distance from the output layer, where loss propagation starts, the embedding layers, particularly the embedding layer of the encoder, are the farthest of all the parameters in the model.
Because the approach is different, the pre-training of the embedding layers can be applied to the model together with the bypass connections.
In our experiment, we employed the B2T connection \cite{takase-etal-2023-b2t} for the bypass connection and found that pre-training of the embedding layers resulted in additional improvement.

The advantages of the pre-training the embedding layers are twofold: high versatility and low cost.
First, most NN-based NLP models contain embedding layers, which makes this method generally applicable.
In text processing, the input to the model is often a sequence of tokens (words, subwords, characters, or bytes), and the first step is to turn it into a sequence of embeddings using an embedding layer: a look-up table that holds the embeddings of the tokens.
While the standard model architecture of NLP models has been changing over the years, for instance from RNNs to Transformers, both architectures contain embedding layers; therefore, pre-training of the embedding layers is always applicable.

Another advantage of this method is its low cost.
The embedding layers are only the tables of embeddings of the tokens registered in the vocabulary, and these embeddings can be obtained from the independent training of embeddings, unlike the other components of the model.
Because the training of embeddings was one of the main topics in previous studies on NNs in NLP, several methods or tools that can train embeddings quickly using CPUs only exist.
Regarding the training data for in-domain pre-training, sharing the training data for the target task is efficient.
Hence, pre-training of the embedding layers is the least costly compared to other partially or fully pre-trained models, which are often pre-trained on a large dataset of the general domain.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.2
\subsection{Pre-training of Embedding Layers}

The procedure for applying the pre-training of the embedding layers is simple: train the embeddings and thereafter initialize the embedding layers of the target model with them before the training.
Figure \ref{fig:overview} shows the overview of the procedure, and as shown in the figure, it actually involves the training of a tokenizer and the tokenization with it.
Popular methods for embedding training are available, such as CBoW, skip-gram \cite{Mikolov2013EfficientEO}, SISG \cite{bojanowski-etal-2017-enriching}, and GloVe \cite{pennington-etal-2014-glove}.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F2
\begin{figure}[t]
\begin{center}
    \includegraphics{31-2ia9f3.pdf}
\end{center}
    \hangcaption{Overview of the process of the pre-training of the embedding layers using the training data of the target task.}
    \label{fig:overview}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



Although most studies have used pre-trained embeddings trained on a large monolingual corpus \cite{lample-etal-2016-neural} or publicly available pre-trained embeddings \cite{kim-2014-convolutional,ma-hovy-2016-end,qi-etal-2018-pre}, in this study, we pre-trained the embeddings using the training dataset of the target MT tasks following a previous study \cite{neishi2017bag}.
The reason for using the target dataset was to avoid the domain discrepancies between pre-training and training.
Because we consider a high-resource situation instead of a low-resource situation in this study, the training dataset is sufficient for pre-training the embeddings.

In addition, we train the tokenizer using the target dataset.
A tokenizer and an NLP model are strongly connected; therefore, publicly available embeddings force users to use the same tokenizer or vocabulary for the target NLP model.
However, it has also been shown that the domain discrepancies in vocabulary are problematic \cite{sato-etal-2020-vocabulary}.
Traning the tokenizer in the target dataset can maintain the vocabulary tailored for the target dataset, which is a better choice in domains that are distant from the general domain.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.3
\subsection{Methods to Train Embeddings}
\label{sec:we-methods}
This section reviews the methods to train embeddings, which we use in this study.
In this section, we refer to the tokens as words because the following methods were originally proposed to train word embeddings at the time, whereas we used subwords as the processing token unit in our experiments.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{CBoW: Continuous Bag-of-Words}
The CBoW method trains word embeddings with the objective of estimating a word in a sequence of words from the context words around the target word, that is, over a given document that has $D$ words in total, the objective is to estimate the target word $w_t$ using the surrounding words $\{w_{t-c}, ..., w_{t-1}, w_{t+1}, ..., w_{t+c}\}$ within the context window size of $c$.
The objective function is expressed as follows:
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{align}
J_\mathrm{CBoW} & = \sum_{i=1}^{D} \log p(w_i | w_{i-c}, ..., w_{i-1}, w_{i+1}, ..., w_{i+c}) 
\end{align}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
To train the embeddings faster, another objective function known as negative sampling (NS) is combined with the objective function above. NS contributes to the quality of the word embeddings of frequent words \cite{NIPS2013_9aa42b31}.
NS is also employed in skip-gram and SISG, which are explained as follows.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Skip-gram}
In contrast to CBoW, the skip-gram method trains word embeddings with the objective of estimating, given a word in a sequence of words, the words surrounding the given word.
Over a given document that has $D$ tokens in total, the objective is to estimate the words within the window size of $c$ $\{w_{t-c}, ..., w_{t-1}, w_{t+1}, ..., w_{t+c}\}$ from the central token $w_t$.
The objective function is expressed as follows:
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{align}
J_\mathrm{skip\mathchar`-gram} & = \sum_{i=1}^{D} \sum_{j=-c, i \neq j}^{c}  \log p(w_j | w_i)
\end{align}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{SISG: Subword Information Skip-Gram}
SISG is an extension of skip-gram that integrates the surficial information of the words into the resulting word embeddings.
The idea is to construct word embeddings by summing the n-gram subwords of a word.
Therefore, SISG uses the same objective function as the original skip-gram method, but is different in the representation of the word embedding; the original skip-gram method holds the word embeddings of each word, but SISG holds the subword embeddings and constructs a word embedding from these subword embeddings.
\rev{Therefore, SISG requires more memory and training time than the original skip-gram.}
This approach is particularly effective for low-frequency, out-of-vocabulary words.
However, because the most recent NLP systems, including our model in this study, employ subword tokens (or smaller tokens) as token units, we trained subword instead of word embeddings in our experiments.
Hence, the benefit of using subword information may decrease.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{GloVe: Global Vectors}
In contrast to the described three methods, which exploit only local information within the context window, GloVe exploits global information over the entire document.
The main idea behind GloVe stems from the co-occurrence counts of words.
GloVe trains word embeddings to estimate the co-occurrence counts from the dot product of the target word embeddings and context word embeddings.
Given a vocabulary size of $V$, the co-occurrence matrix is denoted as $\bm{X}$ whose element $X_{ij}$ denotes the co-occurrence count of the target word $i$ and context word $j$.
\rev{The word embedding of word $i$ is denoted as $e_i$ and the context word embedding of context word $j$ is denoted as $\tilde{e_j}$.}
With the additional bias terms, $b_i$ and $\tilde{b_j}$ for target word $i$ and context word $j$, respectively, the objective function is as follows:
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{align}
J_\mathrm{GloVe} &= \sum_{i,j=1}^{V} f(X_{ij})(\rev{e_i^T \tilde{e_j}} +b_i +\tilde{b_j} - \log (X_{ij}))^2\\
f(x) & =
\begin{cases}
(x / x_{\mathrm{max}})^\alpha & \mathrm{if} \; x < x_{\mathrm{max}} \\
1 & \mathrm{otherwise}
\end{cases}
\end{align}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
where $f(X_{ij})$ denotes a weighting function to soften the impact of the word frequency gap, and $x_{\mathrm{max}}$ and $\alpha$ are hyperparameters that are set to 100 and 3/4 by default, respectively.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4
\section{Experiments}\label{sec:experiments}
To verify whether the in-domain pre-training of the embedding layers also works for the Transformer-based NMT model, we conducted experiments on two MT tasks: ASPEC English-to-Japanese \cite{nakazawa16} and WMT2017 English-to-German \cite{bojar-etal-2017-findings}.
First, we compared four types of methods to pre-train embeddings (\S \ref{sec:comp_methods}).
Next, we trained the NMT models using different learning rates to investigate the effect of pre-training on the training time (\S \ref{comp_learning}).
We further conducted experiments to identify whether it is the encoder's or the decoder's embedding layer that is more influential (\S \ref{comp_enc_dec}), and how the impact changes over the training data size (\S \ref{comp_datasize}).
We also analyzed the embedding layers of the trained NMT models and observed the change of the embeddings by the in-domain pre-training (\S \ref{sec:analysis}).
Additionally, we conducted experiments to investigate the impact of the vocabulary size (\S \ref{sec:vocab}), additional in-domain monolingual data (\S \ref{sec:add_mono}), and the warm-up strategy(\S \ref{sec:warm}) to our proposed method.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.1
\subsection{Settings}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.1.1
\subsubsection{Model}
\label{lab:model}
Our 
NMT model is a Transformer with self-attention layers that employ relative position \cite{shaw-etal-2018-self}, which we implemented using PyTorch (ver.~2.0.0).\footnote{\url{https://pytorch.org}}
To avoid pre-trained parameters from being drastically updated, we discarded the warm-up strategy, which is intended for stable training; instead, we employed post-layer normalization (LN) with a B2T connection \cite{takase-etal-2023-b2t}, which is a type of bypass connection.\footnote{\rev{The original paper of Transformer \cite{vaswani2017attention} employs a warm-up strategy, probably because of the unstable training as a result of post-LN. \citeA{pmlr-v119-xiong20b} showed that their proposed initialization method can realize stable training of a deep transformer model without warm-up and layer normalization. \citeA{pmlr-v119-huang20f} confirmed that the pre-layer normalization can also remove the warm-up, and a B2T connection, which we employ in this study, also contributes to the stability of the training. \revv{We report the experimental results of our method with and without a warm-up strategy in \S \ref{sec:warm}.}}}
We set the hyperparameters to follow the original Transformer \cite{vaswani2017attention}.
Details of the model 
are 
% mod by ynaga to avoid overflowing line by removing ~ between Appendix and \ref
% presented 
listed
in Appendix \ref{sec:ap:hyperparam}.

We also conducted reproductive experiments using an RNN-based NMT (RNN-NMT) model \cite{neishi2017bag}.
Our RNN-NMT model is a long short-term memory (LSTM)-based model with global encoder-decoder attention and input feeding \cite{luong-etal-2015-effective}, which is also implemented in the same manner as the Transformer-based model.
The employment of the B2T connection also aims to keep the setting of the two different structured models as similar as possible because warm-up is not commonly used for RNN-NMT models.
Settings different from those of the Transformer NMT model are listed in 
\ynn{Appendix} \ref{sec:ap:hyperparam-rnn}.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.1.2
\subsubsection{Datasets}\label{sec:dataset}
We conducted experiments on English-to-Japanese (En-Ja) and English-to-German (En-De) translation tasks.
For the En-to-Ja translation task, we used ASPEC \cite{nakazawa16}, which is a parallel corpus made from the abstract sections of scientific papers.
For the En-to-De translation task, we used a dataset from the WMT2017 \cite{bojar-etal-2017-findings} news translation task, which comprises publicly available parallel corpora mainly in the news domain.

For the ASPEC task, we used the first 1.5M sentence pairs because the dataset is sorted by the similarity score \cite{nakazawa16} and the latter half of the ASPEC dataset is 
of poor quality.
For the WMT2017 task, we used preprocessed data distributed on the WMT2017 official site\footnote{\url{https://data.statmt.org/wmt17/translation-task/preprocessed/}} and used newstest2015 and newstest2016 as development and test data, respectively.

\ynn{We preprocessed and tokenized the datasets as follows.}
For the ASPEC dataset, we \ynn{truecased and tokenized} the English data using Moses toolkit (ver.~4.0)\footnote{\url{http://www2.statmt.org/moses/}} \cite{koehn-etal-2007-moses}, but we did not apply any word-level preprocessing to the Japanese data.
Subsequently, we applied subword tokenization using SentencePiece \cite{kudo-2018-subword} with a shared vocabulary size of 16,000 using the unigram language model.
For the WMT2017 task, \yn{because the data were already preprocessed,} we only applied subword tokenization using SentencePiece with a shared vocabulary size of 40,000 using the unigram language model.

\rev{In this study, we did not apply language-specific word-level tokenizers before applying subword-level tokenizers, but directly applied subword-level tokenizers because such language-specific preprocessing not only reduces the portability of the resulting models but also incurs undesired impacts in handling inputs written in other languages.
These issues have been warned of by the developer of SentencePiece,\footnote{\rev{\url{https://qiita.com/taku910/items/fbaeab4684665952d5a9} (in Japanese)}} and the use of language-specific preprocessing is becoming less popular in multilingual tasks \cite{liu-etal-2020-multilingual-denoising,fan2020englishcentric,nllbteam2022language}.}

In all of the following experiments, we filtered out sentence pairs with more than 100 subwords in either of the languages in the training data.
\revv{The statistics of the datasets used in the experiments are summarized in Table \ref{tab:num_sent}.}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T1
\begin{table}[t]
\input{09table01.tex}
    \hangcaption{\rev{Statistics of the datasets used in our experiments.}}
    \label{tab:num_sent}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.1.3
\subsubsection{Pre-training of Embeddings}
As described in \S \ref{sec:we-methods}, we compared four types of word embedding methods.
For all the methods, we used their original implementations, which are publicly available on GitHub: 
word2vec\footnote{\url{https://github.com/tmikolov/word2vec}} for CBoW and skip-gram, 
fastText\footnote{\url{https://github.com/facebookresearch/fastText}} for SISG, 
and GloVe\footnote{\url{https://github.com/stanfordnlp/GloVe}} for GloVe.
In pre-training embeddings using these tools, all hyper-parameters were set to default values for simplicity.

When applying the pre-trained embeddings to the NMT model, we initialized the embedding layer with the pre-trained embeddings instead of the PyTorch default operation, random initialization using a normal distribution.
However, because the hyper-parameter settings of the tools, except GloVe, include a minimum token count to discard the least frequent tokens, 
\yn{embeddings are not acquired for rare tokens.}
For these tokens, including a special token for out-of-vocabulary tokens, we initialized them with the average 
of all acquired embeddings.
On ASPEC, with a vocabulary size of 16,004,\footnote{We prepared four special tokens for padding, start of sequence, end of sequence and unknown token.} we acquired 15,781 embeddings from CBoW, skip-gram, and SISG, and 15,997 embeddings from GloVe.
On WMT2017, with a vocabulary size of 40,004, we acquired 38,153 embeddings from CBoW, skip-gram, and SISG, and 39,979 embeddings from GloVe.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T2
\begin{table}[t]
\input{09table02.tex}
\hangcaption{\rev{Cross entropy losses on the validation datasets.}}
\label{tab:valid_loss}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.1.4
\subsubsection{Evaluation}
We trained the 
NMT models for 300,000 steps and used the cross-entropy loss\footnote{\yn{The cross-entropy loss here is a pure cross-entropy loss and is different from the loss used during the training that employs label smoothing \cite{7780677}.}} on the validation data to select the model for evaluation. We employed an early stopping strategy to stop training if no update on the best validation loss for 20,000 steps was present.
\rev{\yn{Table \ref{tab:valid_loss} lists the average of the \ynn{best} validation losses of three models trained with different random seeds.}}
The best pre-training methods were CBoW 10 on ASPEC and skip-gram 2 and 5 on WMT2017.
In the following experiments, except for those in \S \ref{sec:comp_methods} \ynn{which compared the pre-training methods},
we 
\revv{adopted skip-gram 5 as the pre-training method \ynn{to perform comparisons across datasets}}.
%  under the same settings % removed by ynaga 

For the evaluation, we 
\yn{measured} BLEU \cite{papineni-etal-2002-bleu} using sacreBLEU \cite{post-2018-call} and COMET \cite{rei-etal-2020-comet} \yn{using} the wmt22-comet-da model \cite{rei-etal-2022-comet}.
\rev{All the reported scores are the average scores of the three models trained with the same settings, except for the random seed.}
Note that the pre-training of the embeddings is also run with different seeds such that the models of the same setting with different seeds use different pre-trained embeddings.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T3 % moved by ynaga
\begin{table}
\input{09table03.tex}
\hangcaption{BLEU and COMET scores \ynn{of NMT models} on ASPEC. \rev{The best model on the validation loss is underlined (see Table \ref{tab:valid_loss}).}}
\label{tab:score_aspec}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.2
\subsection{Comparison of Methods to Pre-train Embeddings}
\label{sec:comp_methods}
\ynn{We first clarify} whether pre-training the embedded layers is also effective for a Transformer-based NMT model and which word embedding method is the most suitable for the pre-training.
As concerns the backpropagation flow, Transformer-based models have an advantage over RNN-based models because self-attention layers provide a shorter path in the time direction than RNNs.
However, the Transformer model is known to be difficult to train; therefore bypass connections and learning rate tunings are often applied.
We trained the NMT models whose embedding layers were pre-trained using four types of word embedding methods\yn{, as \ynn{mentioned} in \S \ref{sec:we-methods}. Tables \ref{tab:score_aspec} and \ref{tab:score_wmt}} list the scores on the automatic evaluation measures.
The window sizes for the word embedding training were 2, 5, and 10 for all methods and 15 for GloVe because 15 is its default window size.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T4
\begin{table}
\input{09table04.tex}
\hangcaption{BLEU and COMET scores of \ynn{NMT models} on WMT2017. \rev{Underline indicates the best model on the validation loss (see Table \ref{tab:valid_loss}).}}
\label{tab:score_wmt}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


For both ASPEC and WMT2017 
datasets, % added by ynaga
all word embedding methods exhibited performance gains in both BLEU and COMET scores across all window sizes.
This suggests that no negative impact is present on the pre-training of embedding layers that use objectives other than the MT task.
This might be because pre-training is performed on the same dataset as the target translation task, whereas using a general domain dataset may risk negative % removed a by ynaga (English mistakes)
results in a high-resource situation.

Although no score decrease was obsearved, the gains in the COMET score in ASPEC were smaller than those in WMT2017\@.
This is because the COMET scorer is not trained or tuned for the domain of ASPEC, the domain of scientific papers;
COMET uses XLM-R \cite{conneau-etal-2020-unsupervised} as the base model, which is trained on CommonCrawl \cite{wenzek-etal-2020-ccnet}
and is tuned for the WMT 2022 Metrics Shared Task \cite{freitag-etal-2022-results}, which mainly comprises news domain data.
Therefore, we did not discuss the COMET scores in ASPEC.

As concerns the BLEU score, skip-gram achieves the best gain and SISG achieves the second best on both datasets,
whereas as concerns the COMET score, SISG achieves the best gain and skip-gram is the second best on WMT2017.
Because SISG is an extension of skip-gram and our experiments were conducted with subword-level tokens, 
we may as well assume that skip-gram is the best for Transformer-based NMT models.

This result is different from the that of the RNN-based NMT models \cite{neishi2017bag} where CBoW was the best.
The BLEU score gain was not as high as that in a previous study in which the gain reached 
1.00.
Because the previous study performed only a single run for each of the NMT models, the good performance of the CBoW could be just a coincidence; however, this is unlikely because the CBoW achieves good results over all window sizes.

\revv{\yn{Tables \ref{tab:score_aspec_je} and \ref{tab:score_wmt_de} summarize} the supplementary experimental results for the opposite \ynn{translation direction}, ASPEC Japanese-to-English and WMT2017 German-to-English, respectively. \ynn{The proposed method contributed to the gains in both BLEU and COMET scores of NMT models on the ASPEC and WMT2017 datasets regardless of the translation direction.}}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T5
\begin{table}
\input{09table05.tex}
\hangcaption{\revv{BLEU and COMET scores \ynn{of NMT models} on ASPEC Japanese-to-English \ynn{(the opposite translation direction from Table \ref{tab:score_aspec})}.}}
\label{tab:score_aspec_je}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T6
\begin{table}
\input{09table06.tex}
\hangcaption{\revv{BLEU and COMET scores \ynn{of NMT models} on WMT2017 German-to-English \ynn{(the opposite translation direction from Table \ref{tab:score_wmt})}.}}
\label{tab:score_wmt_de}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.3
\subsection{Impact of Learning Rate \ynn{on} Convergence}
\label{comp_learning}
For the internal structure of the transformer layers in both the encoder and decoder, we employed post-LN with a B2T connection to achieve stable training while expecting better performance.
The motivation for stable training is to exclude warm-up steps, which may ruin the pre-trained parameters.
Therefore, we conducted an experiment to train the NMT models using different learning rates to confirm whether the impact of the pre-trained embeddings was lost by a large learning rate.
\rev{While the default learning rate in this study was 0.0001, we also examined the learning rates of 0.001, 0.0003, and 0.00003.}
As a previous study \cite{neishi2017bag} argued that pre-trained embeddings not only improve the model performance but also contribute to earlier convergence, we compared the pre-trained models with normal models as concerns the convergence time.

\yn{Tables \ref{tab:lr_aspec} and \ref{tab:lr_wmt} list the \ynn{results with different learning rates on} ASPEC and WMT2017, respectively.}
\rev{We did not report the results of a learning rate of 0.001 because it resulted in unstable training on both datasets.}
We expected that the larger learning rate might disrupt the pre-trained parameters and make the method not work,
but it actually turned out that the pre-trained embeddings work even with a large learning rate (as long as the model is stably trained)
as the learning rate of 0.0003 yielded the best BLEU score on ASPEC and the best COMET score on WMT2017.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T7
\begin{table}[t]
\input{09table07.tex}
%%%%%    \tabcolsep 4pt
    \hangcaption{BLEU and COMET scores of NMT models trained with different learning rates on ASPEC.}
    \label{tab:lr_aspec}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T8
\begin{table}[t]
\input{09table08.tex}
    \hangcaption{BLEU and COMET scores of NMT models trained with different learning rates on WMT2017.}
    \label{tab:lr_wmt}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T9
\begin{table}[t]
\input{09table09.tex}
    \hangcaption{\ynn{A}verage best steps of NMT models trained with different learning rates on ASPEC and WMT2017.}
    \label{tab:step_lr}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\ynn{Table \ref{tab:step_lr} summarizes the average best steps for different learning rates on ASPEC and {\linebreak}WMT2017.} % removed a space between WMT2017 and . by ynaga
We can observe that a small learning rate requires more training steps for convergence because smaller learning rates slow the training.
Considering \ynn{the gaps between the average best steps and the given maximum training steps of 300,000},
the models on the ASPEC dataset with learning rates of 0.0003 and 0.0001 seem to have reached convergence 
because the average best steps were more than 30,000 before the given maximum training steps.
For the models on the WMT2017 dataset, it seems that only the model pre-trained by skip-gram with a learning rate of 0.0003 reached convergence.
The average best steps of the other models were 
\ynn{close} to the given maximum training steps.

From a different perspective, we also observed that the average best steps were different for the two datasets.
The values of the models on WMT2017 were larger than those of the models on ASPEC\@.
The datasets differ in language pairs and domains, but the most relevant difference is the size of the dataset.
While the ASPEC dataset contains 1.5M sentence pairs as training data, the WMT2017 dataset contains 5.9M sentence pairs as training data, which is almost four times larger than that of the ASPEC dataset.
Precisely, the quality of the training and validation data would also be relevant; 
however, we do not consider this because the quality estimation of bitexts is another difficult task.

As concerns the gaps in the average best steps between random initialization and pre-training by skip-gram,
the models that seem to have reached convergence show that pre-trained embeddings lead the model to earlier convergence.
On the ASPEC dataset, the model \yn{with embedding layers} pre-trained by skip-gram yielded the best results at 41,333 steps and 32,000 steps earlier than the randomly initialized model, with learning rates of 0.0003 and 0.0001, respectively.
In addition, on the WMT2017 dataset, the model \yn{with embedding layers} pre-trained by skip-gram yielded the best results at 26,000 steps earlier than the randomly initialized model, with a learning rate of 0.0003.
However, the randomly initialized model does not appear to have reached convergence; therefore, the gap would be larger when trained for more steps.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.4
\subsection{Impact of Target Embedding Layers for Pre-training}
\label{comp_enc_dec}
The intuition behind the idea of pre-trained embeddings is that the positions of the embedding layers are the farthest from the output layer; therefore, the back-propagation of the training loss would be the least effective among all the parameters in the model.
This is a common idea, as can be observed in deep NNs with added bypass connections, 
such as residual connection \cite{7780459} or B2T connection\ynn{ \cite{takase-etal-2023-b2t}}, which we employed in our model.
To verify this idea, we pre-trained the embedding layer of the encoder only or that of the decoder only for comparison.
Therefore, we expect the pre-training of the embedding layer of the encoder to yield a larger impact because the embedding layer of the encoder is more distant from the output layer than that of the decoder.


Tables \ref{tab:ed_aspec} and \ref{tab:ed_wmt} summarize \ynn{the results with embedding layers partially pre-trained on ASPEC and WMT2017, respectively}.
As expected, pre-training the embedding layer of the encoder yielded larger gains in both BLEU and COMET scores on both datasets.
The pre-training of the embedding layer of the decoder showed tiny gains in the BLEU score on ASPEC and in the COMET score on WMT2017, but rather negative impacts in the BLEU score on WMT2017.
Considering that both-side pre-training achieves better scores than encoder-side-only pre-training, some positive impact in pre-training the embedding layer of the decoder exists, but this seems to be significantly limited.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T10
\begin{table}
\input{09table10.tex}
    \hangcaption{BLEU and COMET scores of NMT models \yn{with embedding layers partially pre-trained} on ASPEC.}
    \label{tab:ed_aspec}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T11
\begin{table}
\input{09table11.tex}
    \hangcaption{BLEU and COMET scores of NMT models \yn{with embedding layers partially pre-trained}  on WMT2017.}
    \label{tab:ed_wmt}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


Although we demonstrated the significance of the distance from the starting point of the loss propagation, it also indicated that even with the B2T connection, room for improvement exists.
Pre-training is an approach to handle hard-to-reach parameters directly and requires a two-step operation: pre-training and main training.
In contrast, the bypass connection makes the model easy to train and does not require any additional operations.
As concerns efficiency, we may focus more on the latter approach.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.5
\subsection{\ynn{Impact of}
Training Data Size}
\label{comp_datasize}
\citeA{qi-etal-2018-pre} reported that pre-training of the embedding layers is effective when we can construct a moderately working NMT model, that is, when 
\yn{the training data are not excessively small.}
Our 
settings were different from theirs in the dataset for pre-training; we used the same training dataset as the target task for pre-training, while they used a large, fixed-size dataset in the general domain for pre-training.
Therefore, we do not search for the sweet spot where the pre-training exhibits a large impact, but we confirm whether the impact does not vanish as the training data size increases, because our purpose is to improve the NMT model performance in a high-resource setting.

In this experiment, we downsampled the training dataset into 10k, 100k, and 1M sentence pairs\footnote{\rev{Note that the length filtering \ynn{was} applied after the downsampling, \ynn{which removed some sentence pairs from the 10k, 100k, and 1M sentence pairs as shown in Tables \ref{tab:datasize_aspec} and \ref{tab:datasize_wmt}}.}} by sampling from the beginning of the dataset, and used them for both the pre-training of the embeddings and training of the NMT models.
The experimental settings were the same as those used for preceding experiments, indicating that the settings were not tuned for low-resource situations.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T12
\begin{table}[t]
\input{09table12.tex}
    \hangcaption{BLEU \yn{and COMET} scores of NMT models on ASPEC with different training data sizes.}
    \label{tab:datasize_aspec}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T13
\begin{table}[t]
\input{09table13.tex}
    \hangcaption{BLEU \yn{and COMET} scores of NMT models on WMT2017 with different training data sizes.}
    \label{tab:datasize_wmt}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


Tables \ref{tab:datasize_aspec} and \ref{tab:datasize_wmt} summarize the results \yn{with different training data sizes} for ASPEC and WMT2017, respectively.
In all training data sizes and both datasets, except for the training data size of 10k on WMT2017, the pre-trained embeddings provided a score gain
in both the BLEU and COMET scores.
The negative impact of the training data size of 10k on WMT2017 is not surprising because the acquired NMT model is less than moderately working, which agrees with a previous study \cite{qi-etal-2018-pre}.

\yn{The training data size of the NMT models (and its pre-training) affects the impact of the pre-training on the NMT models, and their impact differs across the datasets.}
\rev{On ASPEC, the gain in BLEU is the best at the dataset size of 100k and it becomes smaller at the size of 1M and 1.5M.
In contrast, on WMT2017, the gain in BLEU is large at the dataset size of 100k and it decreases at the size of 1M but reaches the best at the size of 5.8M.
The gain in COMET on WMT2017 achieved a similar result, but the best score is at the size of 100k.
From these results, we can deduce that our proposed method yields a large improvement with a moderate amount of training data, and the improvement becomes small or large depending on the dataset as the training size increases; however, it does not vanish.}
Because the settings used in this experiment were for full use of the training data, better model tuning for low-resource settings will lead to a clearer conclusion.

\rev{The training data size affected the quality of 
the embedding layers of the \ynn{baseline NMT model with embedding layers randomly initialized}
and that of the pre-trained embeddings \ynn{to initialize our NMT models}.
When the training data size is small, the former becomes low, leaving room for improvement using our method, whereas the latter also becomes low, which degrades the impact of our method. In \S \ref{sec:add_mono}, we experimented with additional monolingual data to increase the quality of the pre-trained embeddings \ynn{and maximize the advantage of our method.}}




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.6
\subsection{Impact of Pre-Training on Embeddings after NMT Training}
\label{sec:analysis}
In this section, we analyzed the embedding layers of the NMT models trained on the translation task to investigate \yn{the effect of the pre-training of the embedding layers on the embeddings of the resulting NMT models.}
Specifically, we compared the embeddings of a randomly initialized NMT model with those of an NMT model whose embedding layers were pre-trained \yn{in the following manner.}
First, we aligned the two embedding spaces using a simple linear transformation \cite{mikolov2013exploiting}. Thereafter, \yn{we} inspected the distances between the corresponding embeddings \yn{for identical tokens}.
The distances between the corresponding embeddings indicate the extent to which the embeddings were affected by the pre-training.
For this analysis, we used the best-performing models on ASPEC of the randomly initialized models and the models that were pre-trained by skip-gram with a window size of 5 in \S\ref{sec:comp_methods}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T14 % moved by ynaga
\begin{table}[t]
\input{09table14.tex}
    \hangcaption{Top-10 tokens by the distance of the embeddings of the encoder. Underbar `\_' in the token is a special character to denote a space. Distance is squared distance and count is the token frequency in training data. Tokens with less than 100 counts are excluded.}
    \label{tab:tops_enc}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T15 % moved by ynaga
\begin{table}
\input{09table15.tex}
    \hangcaption{Top-10 tokens by the distance of the embeddings of the decoder. Underbar `\_' in the token is a special character to denote a space. Distance is squared distance and count is the token frequency in training data. Tokens with less than 100 counts are excluded.}
    \label{tab:tops_dec}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Tables \ref{tab:tops_enc} and \ref{tab:tops_dec} list the top-10 tokens as concerns the distance between the two embeddings of the encoder and decoder, respectively.
As \citeA{qi-etal-2018-pre} reported that the pre-trained embeddings improve the accuracy of the translation, particularly for low-frequency words, the tokens that have a large distance are much less frequent than those that have a small distance, except `。' in Table \ref{tab:tops_dec}.
Unsurprisingly, frequent tokens are well aligned because they are more influential in building the embedding space.





Tables \ref{tab:neighbor_enc} and \ref{tab:neighbor_dec} list the nearest neighbor tokens of the top-5 distant tokens in Tables \ref{tab:tops_enc} and \ref{tab:tops_dec}, respectively.
We observed that the embeddings of the randomly initialized model fall into the hubness problem \cite{shigeto2015ridge} in which a certain example becomes the nearest neighbor of several other examples.
In contrast, the embeddings of the model pre-trained by skip-gram have somewhat related tokens as the nearest neighbors, which demonstrates the positive impact of pre-training of the embedding layers.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T16
\begin{table}[t]
\input{09table16.tex}
%%%%%    \tabcolsep 4pt
    \hangcaption{Nearest neighbor tokens of the top-5 distant token embeddings in the encoder.}
    \label{tab:neighbor_enc}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T17
\begin{table}[t]
\input{09table17.tex}
    \hangcaption{Nearest neighbor tokens of the top-5 distant token embeddings in the decoder.}
    \label{tab:neighbor_dec}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.7
\subsection{\rev{Impact of Vocabulary Size}}
\label{sec:vocab}
\rev{As mentioned in the previous section, we have confirmed that our proposed method is particularly effective for low-frequency tokens in \S \ref{sec:analysis}.
Based on this, we can expect that our method will result in larger improvements with a larger vocabulary size, because a larger vocabulary size allows less frequent tokens to be maintained in the vocabulary.
To examine this hypothesis, we trained NMT models of different sizes: a vocabulary size of 40,004 for ASPEC and a vocabulary size of 16,004 for WMT2017.}

\rev{Tables \ref{tab:vocab_aspec} and \ref{tab:vocab_wmt} summerize the results on ASPEC and WMT2017, respectively.
For the score gain, larger vocabulary sizes resulted in larger score gains in both BLEU and COMET scores on both datasets, as expected.
However, regarding the score, the smaller vocabulary size, which is our default setting, yielded a higher BLEU and COMET score on ASPEC\@.
This indicates that our method is effective with a larger vocabulary size; however, setting a proper vocabulary size for the target task dataset is more important.}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T18
\begin{table}
\input{09table18.tex}
    \hangcaption{\rev{BLEU and COMET scores of NMT models with different vocabulary sizes on ASPEC.}}
    \label{tab:vocab_aspec}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T19
\begin{table}
\input{09table19.tex}
    \hangcaption{\rev{BLEU and COMET scores of NMT trained with different vocabulary sizes on WMT2017.}}
    \label{tab:vocab_wmt}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\revv{Here, we briefly discuss the possible impact of using a language-specific word-level tokenizer.
In this study, we did not employ language-specific word-level tokenizers, as mentioned in \S \ref{sec:dataset}.
A subword-level tokenizer results in a vocabulary that includes \yn{more} low-frequency tokens, for which our method is effective, because the tokens are not allowed to include word boundaries.
From the results of this section, we assume that 
vocabulary size would have a greater impact than 
word-level tokenizers.}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.8
\subsection{\rev{Impact of Additional Monolingual Data}}
\label{sec:add_mono}
\rev{As mentioned in \S \ref{comp_datasize}, we used the same dataset to pre-train the embedding layers and train the NMT model.
However, in realistic situations, we expect additional monolingual data from the target domain.
Reflecting that our method is effective for low-frequency tokens, we can expect that additional monolingual data will help train low-frequency tokens better, making our method more effective.
In this section, we examine the impact of additional monolingual data using a pseudo-monolingual dataset created from a given bitext.}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T20 % moved by ynaga
\begin{table}
\input{09table20.tex}
    \hangcaption{\rev{BLEU and COMET scores of models trained on 100k sentence pairs of ASPEC.}}
    \label{tab:add_mono_aspec}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T21 % moved by ynaga
\begin{table}
\input{09table21.tex}
    \hangcaption{\rev{BLEU and COMET scores of models trained on 100k sentence pairs of WMT2017.}}
    \label{tab:add_mono_wmt}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\rev{To build a dataset that comprises bitext data and pseudo-monolingual data, we split the given training dataset into two for both the ASPEC and WMT2017 datasets, using the first 100k sentence pairs in the original bitext as the bitext data and the remaining as the pseudo-monolingual data.
To build the pseudo-monolingual data and promote the reproducibility of our results, we assigned numbers to the remaining parts and picked the source sentences that had odd numbers and the target sentences that had even numbers such that the resulting pseudo-monolingual data did not become a comparable corpus.
The resulting pseudo-monolingual data of ASPEC \yn{and WMT2017 comprised} 1,454,250 \yn{and} 2,876,229 
unaligned sentences for the source and target languages, respectively.
We pre-trained the embeddings using both the 100k bitext data and the pseudo-monolingual data, and thereafter trained the NMT model using the 100k bitext data.}

% line is recovered by ynaga
\rev{Tables \ref{tab:add_mono_aspec} and \ref{tab:add_mono_wmt} list the BLEU and COMET scores of the NMT models, respectively.
In both datasets, the additional monolingual data yielded score gains for both BLEU and COMET, as expected.
However, the results differ by the dataset; the gain from the additional monolingual data is only +0.12 BLEU score on ASPEC, whereas it is +1.29 BLEU score on WMT2017.
This would be because of the instability of the NMT model; the skip-gram 5 model on ASPEC had a standard deviation of $\pm$0.52 on the BLEU score, which is larger than those of the others.
From the results, we 
conclude that the additional monolingual data of the target domain have a significant impact on our proposed method.}





%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.9
\subsection{\rev{Impact of Warm-up Strategy}}
\label{sec:warm}
\revv{As described in \S \ref{lab:model}, we did not employ a warm-up strategy to make the conditions of the RNN-NMT and the Transformer models as similar as possible.
However, in \S \ref{sec:relatedwork}, we also mentioned that the warm-up strategy is a method to stabilze training, which affects our proposed method.
In this section, we train the models using the warm-up strategy of the original Transformer \cite{vaswani2017attention} with the warm-up steps of 4,000 to observe the impact of the warm-up strategy on our method. Note that the model with the warm-up strategy also employed post-LN with a B2T connection.}

\revv{Tables \ref{tab:warm_aspec} and \ref{tab:warm_wmt} summarize the results for ASPEC and WMT2017, respectively.
The scores of those without a warm-up were slightly better than those with a warm-up on the BLEU score and slightly worse on the COMET score.
Regarding the score gains from our method, we can observe a slight decrease, except for the COMET score on WMT2017.
Based on the small difference in the score gains, we can conclude that the warm-up strategy did not affect our method.}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T22
\begin{table}
\input{09table22.tex}
    \hangcaption{\revv{BLEU and COMET scores of NMT models trained \yn{with and without warm-up} strategy on ASPEC.}}
    \label{tab:warm_aspec}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T23
\begin{table}
\input{09table23.tex}
    \hangcaption{\revv{BLEU and COMET scores of NMT models trained \yn{with and without} warm-up strategy on WMT2017.}}
    \label{tab:warm_wmt}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5
\section{Conclusions}
In this study, we revisited the pre-training of the embedding layers in Transformer-based NMT models.
Experiments using four types of word embedding methods demonstrated the positive impact of pre-training across all the methods, and skip-gram (and SISG) was the best.
Through experiments on different learning rates, we confirmed that the pre-training also contributed to the earlier convergence of the model.
We also verified that the impact of the pre-training is mostly as a result of the pre-training of the embedding layer of the encoder through an experiment to apply the pre-training to either the encoder or the decoder.
To improve the NMT models in high-resource settings, experiments with different training data sizes demonstrated that the impact of the pre-trained embeddings did not vanish as the training data size increased.
An analysis of the embeddings revealed the large impact of the pre-training on the low-frequent tokens.



% Start Acknowledgment
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment
This work was partially supported by JSPS KAKENHI Grant Number JP21H03494.

% Start bibliography
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.7}
    \input{09aaa_bbl.tex}

\clearpage%%%%%
\appendix
\section{Hyperparameters of the Transformer-based NMT Model}
\label{sec:ap:hyperparam}
\ynn{The model settings of the Transformer-based NMT Model with self-attention with relative position \cite{shaw-etal-2018-self} are as follows.}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%TS1
\begin{table}[ht]
\input{09tableS1.tex}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\clearpage%%%%%
\section{Hyperparameters of the RNN-based NMT Model}
\label{sec:ap:hyperparam-rnn}
\ynn{The model settings of the RNN-based NMT model \cite{luong-etal-2015-effective} are as follows.}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%TS2
\begin{table}[ht]
\input{09tableS2.tex}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}
\bioauthor[:]{Masato Neishi}{%
He received his B.A. in Language and Culture in 2015, his B.S. in Electrical and Electronic Engineering in 2017, and his M.S and Ph.D. in Information Science and Technology in 2019 and 2024, respectively, from the University of Tokyo.
He is currently working as a research engineer at Mirai Translate, Inc.
His research interests include natural language processing, in particular machine translation, and linguistics.}

\bioauthor[:]{Naoki Yoshinaga}{%
He received his Ph.D. from the University of Tokyo in 2005. He has been JSPS Research Fellow (DC1, PD) from 2002 to 2008, and Associate Professor at the University of Tokyo since 2016. His research interests include computational linguistics and machine learning, in particular efficient and adaptive natural language processing in the wild.}
\end{biography}

 \biodate   %%%% Following settings will be edited by editors.


\end{document}
