\documentclass[english]{jnlp_3.5}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{hangcaption_jnlp}
%%%%\usepackage{udline}
%%%%\setulminsep{1.2ex}{0.2ex}
%%%%\let\underline=\ul
\allowdisplaybreaks


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
\usepackage{bm}
\usepackage{multirow}
%%%%\usepackage{url}
%%%%\usepackage{CJKutf8}
\usepackage{booktabs}
\usepackage{xcolor}
%%%%\usepackage{caption}
%%%%\usepackage{subcaption}
\usepackage{arydshln}
\definecolor{htmlgreen}{rgb}{0.0, 0.5, 0.0}
%%%%\newcommand{\revise}[1]{\textcolor{black}{#1}}
%%%%\newcommand{\final}[1]{\textcolor{black}{#1}}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{29}
\Number{2}
\Month{June}
\Year{2022}
%%%%\Issuetitle{Special Issue: XXXXXXXX}

\received{2021}{10}{7}
\revised{2022}{2}{15}
%%%%\rerevised{2006}{5}{6}
\accepted{2022}{3}{23}

\setcounter{page}{611}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Cross-Lingual Transfer Learning\\for End-to-End Speech Translation}
\eauthor{Shuichiro Shimizu\affiref{Author_1} \and Chenhui Chu\affiref{Author_1} \and Sheng Li\affiref{Author_2} \and Sadao Kurohashi\affiref{Author_1}} 
\eabstract{%
End-to-end speech translation (ST) is the task of directly translating source language speech to target language text. It has the potential to generate better translation than those obtained by simply combining automatic speech recognition (ASR) with machine translation (MT). We propose cross-lingual transfer learning for end-to-end ST, where the model parameters are transferred from the ST pretraining stage for one language pair to the ST fine-tuning stage for another language pair. Experiments on the CoVoST 2 and multilingual TEDx datasets in many-to-one settings show that our model outperforms the model that uses English ASR pretraining by up to $2.3$ BLEU points. Through an ablation study investigating which layer of the sequence-to-sequence architecture contains important information to transfer, it was demonstrated that the lower layers of the encoder contain language-independent information for cross-lingual transfer. Extensive studies were conducted on (1) ASR pretraining language, (2) ST pretraining language pair, (3) multilingual methods, and (4) model sizes. It was demonstrated that (1) Using the same language as the ASR pretraining language and the ST fine-tuning source language results in good performance. (2) A high-resource language pair is a good choice for the ST pretraining language pair. (3) The proposed method works well in conjunction with multilingual methods. (4) The proposed method can operate with different model sizes.
}
\ekeywords{Speech Translation, End-to-End Speech Translation, Speech-to-Text Translation, Transfer Learning}

\headauthor{Shimizu et al.}
\headtitle{Cross-Lingual Transfer Learning for End-to-End ST}

\affilabel{Author_1}{}{Graduate School of Informatics, Kyoto University}
\affilabel{Author_2}{}{National Institute of Information and Communications Technology}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}
\label{sec:introduction}

Speech translation (ST) is the task of translating speech in one language into text in another language.
There are two main approaches to this task: the cascaded approach \cite{stentiford1988st}, in which automatic speech recognition (ASR) and machine translation (MT) are chained together, and the end-to-end approach \cite{duong2016attentional,berard2016lat}, in which a single sequence-to-sequence model translates directly audio signals into the target text.

The cascaded approach has the problem of error propagation such that possible errors produced by ASR are carried forward to MT without correction.
There is also a loss of prosody information during ASR because texts cannot retain that kind of information.
In contrast, the end-to-end approach does not have an error propagation problem because the target text is produced directly, and also prosody information can be used during translation.
Therefore, the end-to-end approach has recently gained popularity \cite{sperber2020survey,weiss2017sequence}.

Despite these advantages, the end-to-end approach still suffers from the problem of data scarcity, which can lead to low translation performance.
ST corpora contain at most several hundred hours of speech or hundreds of thousands of parallel sentences \cite{digangi2019mustc}, whereas ASR corpora contain a thousand hours of speech \cite{panayotov2015libri} and MT corpora contain billions of parallel sentences \cite{schwenk2021ccmatrix}.

One methods of addressing data scarcity in end-to-end ST, is transfer learning from ASR \cite{bansal2019preasr,stoian2020analyzing} or MT \cite{jia2019leveraging}.
With this method, the pretrained ASR encoder or MT decoder parameters are used to initialize the ST encoder or decoder, which are further fine-tuned on the ST task.
In transfer learning, transferring the parameters between similar tasks is better than transferring them between dissimilar tasks \cite{rosenstein2005transfer}.
The difference between ASR and ST tasks (or MT and ST tasks) leaves room for further improvement.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1
\begin{figure}[b]
\begin{center}
\includegraphics{29-2ia13f1.pdf}
\end{center}
\hangcaption{Overview of the proposed method. Previous studies only applied ASR pretraining for ST (without the red-backgrounded part). The proposed method further applies ST pretraining.}
\label{fig:proposed}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

To address the above problem, we propose cross-lingual transfer learning for end-to-end ST by transferring the parameters of ST models from one language pair to another.
That is, an ST model is trained for one language pair at the pretraining stage, and the model is subsequently trained on another language pair in the fine-tuning stage (see stages 2 and 3 in Figure \ref{fig:proposed}).
Transfer learning is performed between two ST tasks, which is possibly better than transferring from an ASR or an MT task.

Experiments were conducted on the CoVoST 2 dataset \cite{wang2021covost2} and multilingual TEDx dataset \cite{salesky2021mtedx} in many-to-one settings, where English was the target language.
Our model outperformed the model that uses English ASR pretraining by up to $2.3$ BLEU points.

An ablation study was conducted to determine the layer of the sequence-to-sequence architecture containing important information for transfer.
Some of the layers were frozen during the ST fine-tuning stage.
It was determined that the lower layers of the encoder contain language-independent information in many-to-one settings.

Extensive studies were conducted on (1) ASR pretraining language, (2) ST pretraining language pair, (3) multilingual methods, and (4) model sizes.
It was demonstrated that:
\begin{enumerate}
    \item Using the same language as the ASR pretraining language and the ST fine-tuning source language exhibited good performance.
    \item A high-resource language pair is a good choice for the ST pretraining language pair.
    \item The proposed method works well when used with multilingual methods.
    \item The proposed method can work with different model sizes.
\end{enumerate}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Work}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{End-to-End ST}

End-to-end ST is an active research area because of its potential to reduce error propagation and exploit prosody information \cite{sperber2020survey}.
The direct modeling of aligning speech to text was first proposed \cite{duong2016attentional}, and then it was applied to end-to-end ST \cite{berard2016lat}.
Although the end-to-end system has good potential, cascaded systems often outperformed end-to-end systems until recently (e.g., \cite{salesky2020phone}).
Some recent work shows the superiority of the end-to-end systems over the cascaded ones \cite{li2021multilingual}, and the performance gap between cascaded and end-to-end systems is now closing \cite{bentivogli2021versus}.

Transfer learning is one of the major ways to address the data scarcity issue in end-to-end ST.
Attempts have been made to use transfer learning from ASR \cite{bansal2019preasr} and MT.
There are several methods to perform transfer learning from MT, such as explicit transfer (i.e., transfer model parameters) through weakly supervised training \cite{jia2019leveraging}, and implicit transfer through knowledge distillation \cite{liu2019kd} or sequential knowledge distillation \cite{inaguma2021seqkd}.
ASR encoder training has proven to be effective, but MT decoder pretraining (i.e., explicit transfer learning) has not shown to be as effective unless additional adapter components are used \cite{bahar2019comparative}.
The emergence of multilingual ST corpora (such as one-to-many corpus \cite{digangi2019mustc}, many-to-one and one-to-many corpora \cite{wang2021covost2}, and non-English-centric corpus \cite{salesky2021mtedx}) led to studies about multilingual ST using language tags \cite{inaguma2019multilingual}, dual decoder \cite{le-etal-2020-dual}, adapter modules \cite{le2021lightweight}, and pretrained models \cite{li2021multilingual}.
Although implicit knowledge transfer might happen in multilingual ST, a further step is required to conduct cross-lingual transfer explicitly.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Transfer Learning for Multilingual MT}

Transfer learning is a learning framework that aims to improve the learning of the target predictive function using the knowledge in the source domain and tasks \cite{pan2010transfersurvey}.
This has shown to be effective in the field of multilingual MT, especially when the knowledge is transferred from a high-resource language pair to a low-resource language pair \cite{raj2020survey}. Previous work has focused on transferring lexical (e.g., \cite{zoph2016transfer}) and syntactic knowledge (e.g., \cite{rudramurthy19}). In addition, language relatedness has been studied in transfer learning for multilingual MT (e.g., \cite{Y17-1038}). To the best of our knowledge, our work is the first to transfer speech knowledge in multilingual MT.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Preliminaries}

We formulate both end-to-end ST and ASR as speech-to-text tasks because they both convert speech input into text output.
Figure \ref{fig:speechToText} presents an overview of the speech-to-text task during inference.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 2
\begin{figure}[t]
\begin{center}
\includegraphics{29-2ia13f2.pdf}
\end{center}
\hangcaption{Overview of the speech-to-text task (both ASR and ST). The figure shows how the model works during inference.}
\label{fig:speechToText}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

In a speech-to-text task, there is a set of speech text pairs, denoted as $\mathcal{S} = \{ (\bm{s}^{(k)}, \bm{t}^{(k)}) \}_{k=1}^K$, comprising a source speech $\bm{s}$ and target text $\bm{t}$, where $K$ is the number of training utterances in the ASR or ST corpus. Hereafter, the superscript $k$ is dropped for conciseness.
$\bm{s}$ and $\bm{t}$ are from the same language if the task is ASR, but from different languages if the task is ST.
Here, $\bm{s} = (s_1, \dots , s_M)$ is the sequence of acoustic signals in the utterance, and $\bm{t} = (t_1, \dots t_N)$ is the sequence of tokens in the utterance, where $M$ is the number of audio samples in the utterance and $N$ is the number of tokens in the utterance.
Each utterance usually corresponds to one sentence but can correspond to two or more sentences.

The sequence of audio signals $\bm{s}$ is processed into a set of speech features $\bm{s}'$ (i.e., a filterbank \cite{purwins2019dlasp}).
First, a sequence of frames with a constant length and stride is obtained from the original signal.
The discrete Fourier transform, Mel scale conversion, and log conversion are then sequentially applied to the frames, yielding a set of $d$-dimensional vectors, called a $d$-dimensional log-Mel filterbank \cite{fayek2016speech}.
The vectors are fed into a one-dimensional convolutional subsampler (i.e., CNN) before they are fed into a sequence-to-sequence architecture.

The objective of training a sequence-to-sequence model is to maximize the log-likelihood
\begin{equation}
    L = \sum_{(\bm{s}', \bm{t}) \in \mathcal{S}'} \log P (\bm{t} | \bm{s}'),
\end{equation}
where $\mathcal{S}' = \{ (\bm{s}', \bm{t}) \}$.
A target utterance is predicted one token at a time:
\begin{equation}
    P (\bm{t} | \bm{s}') = \prod_{i=1}^N P(t_i | t_{<i}, \bm{s}'),
\end{equation}
where $t_i$ is the predicted token and $t_{<i}$ are the previously predicted tokens.

The Transformer \cite{vaswani2017attention} is one of the most popular sequence-to-sequence architectures for speech-to-text tasks, where encoder layers compute the self-attention of a source sequence and decoder layers compute the cross-attention between a source and a target sequence.
The $N_e$ encoder layers and $N_d$ decoder layers are stacked to constitute an encoder and a decoder, respectively.
Positional encoding is applied to the embedding vectors before they are fed into the encoder and the decoder, so that the model can capture the order of the sequence.
An encoder layer comprises a self-attention layer, a feed-forward layer, and a normalization layer.
A decoder layer comprises a self-attention layer, a cross-attention layer, a feed-forward layer, and a normalization layer.
The output of the final decoder layer leads to a linear layer and a softmax layer to produce the output probability distribution.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Cross-Lingual Transfer Learning}

An overview of the proposed method is shown in Figure \ref{fig:proposed}.
Three speech-text pairs were used: 
$\mathcal{S}_1 = \{ (\bm{s}_1, \bm{t}_1) \}$,
$\mathcal{S}_2 = \{ (\bm{s}_2, \bm{t}_2) \}$,
and $\mathcal{S}_3 = \{ (\bm{s}_3, \bm{t}_3) \}$.
$\mathcal{S}_3$ is the main target of translation.
$\mathcal{S}_1$ is a pair for ASR, which means $\bm{s}_1$ and $\bm{t}_1$ are from the same language.
$\mathcal{S}_2$ and $\mathcal{S}_3$ are for ST, which means $\bm{s}_2$ and $\bm{t}_2$, and $\bm{s}_3$ and $\bm{t}_3$ are from different languages.
We transfer parameters from a language pair, $\mathcal{S}_2 = \{ (\bm{s}_2, \bm{t}_2) \}$, to another one, $\mathcal{S}_3 = \{ (\bm{s}_3, \bm{t}_3) \}$.
The focus is on many-to-one translation, which means that $\bm{s}_2$ and $\bm{s}_3$ are from different languages, whereas $\bm{t}_2$ and $\bm{t}_3$ are from the same language.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Stage 1: ASR Pretraining}

ASR is first trained using $\mathcal{S}_1$ (e.g., En--En in Figure \ref{fig:proposed}) to initialize the ST encoder because pretraining on high-resource ASR can improve low-resource ST \cite{bansal2019preasr}.
The pretrained encoder accounts for most of the improvements in this method, and using only the encoder is a widely accepted approach to improving ST performance \cite{wang2021covost2,salesky2021mtedx}; therfore, only the encoder parameters are transferred from stage 1 to stage 2.
The training is stopped when the loss on an ASR development set of $\mathcal{S}_1$ does not improve for a predefined number of epochs.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Stage 2: ST Pretraining}

The pretrained ASR encoder of stage 1 is loaded, and the ST model is trained using $\mathcal{S}_2$ (e.g., Fr--En in Figure \ref{fig:proposed}).
In other words, the model parameters for the CNN, self-attention, feed-forward, and normalization layers in the ASR encoder at stage 1 are passed to the ST encoder at this stage for initialization.
Subsequently, the ST training is performed using $\mathcal{S}_2$.
The training is stopped when the loss on an ST development set of $\mathcal{S}_2$ does not improve for a predefined number of epochs.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Stage 3: ST Fine-tuning}

Finally, the pretrained ST encoder and decoder from stage 2 are loaded, and the ST model is trained on another language pair $\mathcal{S}_3$ (e.g., Es--En in Figure \ref{fig:proposed}).
Here, please note that the language pair already used in stage 2 is not used in stage 3.
In the initialization of the ST encoder, the CNN, self-attention, feed-forward, and normalization layers are shared, as in stages 1 to 2.
For the decoder, in addition to the self-attention, cross-attention, feed-forward, and normalization layers, the vocabulary also needs to be shared with stage 2, to continue training using the same structure.
The same vocabulary as that of stage 2 is used in stage 3 because the target language is always the same in the experiments.
The CNN layers before the encoder, and the linear and softmax layers after the decoder, are also initialized with those at stage 2.
After initialization, the ST fine-tuning is performed using $\mathcal{S}_3$.
The training is stopped when the loss on an ST development set of $\mathcal{S}_3$ does not improve for a predefined number of epochs.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[b]
\vspace{-0.5\Cvs}
\input{13table01.tex}
\hangcaption{The number of utterances for each language pair in the CoVoST 2 and the multilingual TEDx (mTEDx) datasets used in our ST experiments.}
\label{tab:dataset}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experiments}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Datasets}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{CoVoST 2}

In the experiments, CoVoST 2 \cite{wang2021covost2}, a large-scale multilingual ST dataset that covers translations from various languages to English (many-to-one) and from English to various languages (one-to-many) was used.
The part for the many-to-one translation task was used and $13$ languages in addition to English was chosen, following previous work \cite{wang2020fairseqs2t}.
The main reason we conducted experiments on many-to-one settings is to facilitate the case study.
The number of utterances for each language pair is described in Table \ref{tab:dataset}.
The results of the high-resource language pairs are reported in Table \ref{tab:dataset} and the results for low-resource language pairs are presented in Appendix \ref{sec:appendixB}.
The English ASR data included in the dataset was also used. The statistics for the ASR data is shown in Appendix \ref{sec:appendixA} (Table \ref{tab:covost2_asr_stat}).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Multilingual TEDx}

The multilingual TEDx (mTEDx) dataset \cite{salesky2021mtedx}, which includes ASR data in $8$ languages and ST data in $13$ language pairs, was also used in this study.
Although the CoVoST 2 dataset includes many language pairs, most of them contain only one or two hours of speech, and the performance of our proposed method is unclear when such language pairs are used.
Therefore, experiments were also conducted on this dataset, where the language pairs contained at least 11 hours of speech.
Although the mTEDx dataset contains various source or target languages other than English, we focus on translation tasks where the target language is English.
We also focus on high-resource language pairs shown in Table \ref{tab:dataset}, and report the results of low-resource language pairs in Appendix \ref{sec:appendixB}.
The number of utterances for each language pair we used is described in Table \ref{tab:dataset}.
The statistics of the ASR data is shown in Appendix \ref{sec:appendixA} (Table \ref{tab:mtedx_asr_stat}).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{ST Experiments}
\label{sec:cross_exp}

The baseline of our method is to use English as the ASR source language, and then conduct ST fine-tuning on a language pair \cite{wang2020fairseqs2t}.\footnote{We consider the baseline as stage 1 and stage 3 in the proposed method. In fact, only encoder parameters are initialized, and the training can be regarded as stage 2 for further experiments.}
We compare this with the proposed method, where we first train English ASR, then train ST using one language pair, and then fine-tune the model using another language pair.
These experiments were conducted for both the CoVoST 2 and mTEDx datasets.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Settings}

Different settings were applied for the CoVoST 2 and the mTEDx datasets, following the specifications in \cite{wang2021covost2} for the CoVoST 2 experiments and in \cite{salesky2021mtedx} for the mTEDx experiments.

The common settings for both datasets are as follows.
The Fairseq \cite{ott2019fairseq,wang2020fairseqs2t} toolkit was used for preprocessing, training, and decoding.
For preprocessing,
$80$-dimensional log-Mel filterbank features with frames of $25$-ms length and $10$-ms stride were extracted.
Utterance-level cepstral mean variance normalization (CMVN) was applied to the features.
SpecAugment \cite{park2019specaug} with LB policy was applied, to avoid overfitting \cite{bahar2019using}.
The training was parallelized over (simulated\footnote{When only $4$ or $2$ GPU cards are available, we multiplied the \texttt{update-freq} parameter of Fairseq by $2$ or $4$, respectively, so that they simulate $8$ GPU cards. We used V100, GeForce GTX 1080 Ti, and TITAN X GPUs.}) $8$ GPU cards. The maximum number of tokens in a batch was set to be $40,000$.
Decoding was performed using beam search with a beam size of $5$. The utterances of invalid sizes were skipped.
For evaluation, SacreBLEU \cite{post2018call} was used to calculate BLEU\footnote{\texttt{nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.0.0}} (the higher the better) \cite{papineni2002bleu}, chrF2\footnote{\texttt{nrefs:1|case:mixed|eff:yes|nc:6|nw:0|space:no|version:2.0.0}} (character $n$-gram F-score, the higher the better) \cite{popovic-2015-chrf}, and TER\footnote{\texttt{nrefs:1|case:lc|tok:tercom|norm:no|punct:yes|asian:no|version:2.0.0}} (Translation Edit Rate, the lower the better) \cite{snover-etal-2006-ter} scores.
Statistical significance tests \cite{koehn2004statistical} were conducted using SacreBLEU by randomly sampling outputs for $1,000$ times.
The number of samples was set to be the same as the number of test utterances in each dataset.
One system was regarded as significantly superior to the other when the $p$-value is smaller than $0.05$.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{CoVoST 2}
In the CoVoST 2 experiments, the parameters were set according to \cite{wang2021covost2}.
For text preprocessing, SentencePiece \cite{kudorichardson2018sentencepiece} was used to construct a character vocabulary for each language.
For training, we used Transformer with $12$ encoder layers and $6$ decoder layers, where the hidden dimension size was $256$ and the number of attention heads was $4$.
Cross-entropy loss with label smoothing was used as the loss function.

For ASR training (stage 1), the Adam optimizer with an inverse-square-root learning rate scheduler was adopted.
The learning rate increased linearly from $0$ to $0.001$ in $10,000$ updates and then decayed based on the inverse square root of the number of updates.
The training was stopped after $100,000$ updates and the parameters were averaged over the last $10$ epochs.\footnote{We followed \url{https://github.com/pytorch/fairseq/blob/dea66cc294a18dd4d9e59aa0af8d51f951e83884/examples/speech_to_text/README.md}.}
For ST pretraining (stage 2), we also used the Adam optimizer with an inverse-square-root learning rate scheduler; however, the learning rate increased to $0.002$ and then decayed.
The training was stopped after observing no improvement in respect of the development set loss over $10$ epochs, and the best checkpoint was used for evaluation (for the baseline evaluation) or for the following stage (for the experiments of our proposed method).
For ST fine-tuning (stage 3), the same optimizer and the learning rate scheduler as in stage 2 were used.
The optimizer (the learning rate scheduler, meters, and DataLoader) was reset when starting this stage.
The training was stopped after observing no improvement in respect of the development set loss over $10$ epochs, and the best checkpoint was used for evaluation.

For the CoVoST 2 experiments, the language pair at stage 2 was Fr--En, which is the highest-resource language pair in the corpus.
We conduct an extensive study about what language pair to choose at stage 2 using the mTEDx corpus.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{mTEDx} \label{mtedxconfig}
The settings in \cite{salesky2021mtedx} were adopted for the mTEDx experiments.
\pagebreak
Because there are no English ASR data in the mTEDx dataset, the CoVoST 2 English data were used for stage 1.
For text preprocessing, a unigram vocabulary with a vocabulary size of $1,000$ was used for each language.
For training, we used Transformer with $6$ encoder layers and $3$ decoder layers, where the hidden dimension size was $256$ and the number of attention heads was $4$.
Cross-entropy loss with label smoothing (smoothing parameter $0.1$) was used as the loss function.

For all the training of stage 1, stage 2, and stage 3, the Adam optimizer was used with an inverse-square-root learning rate scheduler, where the learning rate increased to $0.002$ for $10,000$ updates and then decayed.
Dropout was applied with the dropout parameter of $0.3$.
The gradients were clipped with the threshold of $10.0$.
The training was stopped after observing no improvement in respect of the development set loss over $10$ epochs, or when the training step had reached $60,000$, or when the epochs had reached $200$.
The checkpoints were averaged over the last $10$ epochs and used for evaluation or for the following stage.

Four high-resource language pairs were selected for the language pair to use at stage 2: Es--En, Pt--En, Fr--En, and It--En.
The low-resource language pairs (Ru--En and El--En) were not used because there would be little knowledge to transfer from a low-resource language pair to a high-resource language pair.
We report the results where we used Es--En as the main results in the following section (\ref{sec:main_results}), and report other results in section \ref{sec:investigate_s2}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 2
\begin{table}[b]
\input{13table02.tex}
\hangcaption{BLUE, chrF2, and TER scores of cross-lingual ST experiments on the CoVoST 2 test set when using Fr as $\bm{s}_2$. The rows with ``B'' are the baseline results, and the rows with ``P'' are the results of the proposed method. $\dag$: The score is significantly superior to the other score ($p < 0.05$).}
\label{tab:covost2Cross}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Results} \label{sec:main_results}

Table \ref{tab:covost2Cross} shows the results on the CoVoST 2 dataset.
In comparing the baseline (without $\mathcal{S}_2$) with the proposed method (with $\mathcal{S}_2$), the proposed method outperformed the baseline for the two high-resource language pairs.
Especially for Es--En, the BLEU score was improved by $1.6$ points.

Table \ref{tab:mtedxCross} lists the results for the mTEDx dataset.
Significance testing showed that our results are significantly superior to the baseline results for Pt--En and It--En.
Especially for It--En, the BLEU score improved by $2.3$ points.
For Fr--En, the performance was comparable to that of the baseline.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[t]
\input{13table03.tex}
\hangcaption{BLEU, chrF2, and TER scores of cross-lingual ST experiments on the mTEDx test set when using Es as $\bm{s}_2$. The rows with ``B'' are the baseline results, and the rows with ``P'' are the results of the proposed method. $\dag$: The score is significantly superior to the other score ($p < 0.05$).}
\label{tab:mtedxCross}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

It might be possible that the language similarity between the source language in stages 2 and 3 influences the ST performance,
considering the fact that the transfers from Fr--En to Es--En or from Es--En to It--En are showing the best performance in the CoVoST 2 and the mTEDx dataset, respectively, and that French, Spanish, and Italian are all Romance languages.
However, the improvement was not observed in the mTEDx Fr--En data using our proposed method, which does not support this assumption.
Language similarities are difficult to define, and further exploration is required.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Manual Evaluation}

We manually checked the translation quality of the CoVoST 2 Es--En and the mTEDx Pt--En corpora.
We sampled $50$ sentences from the test set of each corpus and checked the translations of the baseline and the proposed methods.
We evaluated the translations in terms of adequacy and fluency on a scale of $0$ to $5$.
For adequacy, we checked whether the meaning of the predicted utterances conveys the meaning of the reference utterances.
Regarding fluency, we checked the naturalness of the translations.
The mean and standard deviation of the scores were then calculated.

The results are presented in Table \ref{tab:case_study_scores}.
The translations generated by our proposed method were more accurate and natural than the baseline translations, as determined by the scores.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 4
\begin{table}[tb]
\input{13table04.tex}
\hangcaption{Human-evaluated adequacy and fluency of the predicted translations on the samples from the CoVoST~2 Es--En and the mTEDx Pt--En corpora. The scores are evaluated on a scale of $0$-$5$ and shown as (mean) $\pm$ (standard deviation).}
\label{tab:case_study_scores}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 5
\begin{table}[tb]
\input{13table05.tex}
\hangcaption{Translation examples from the CoVoST 2 Es--En and the mTEDx Pt--En corpora. The transcriptions and references are included in the corpora, and the others are predicted by the models. The same color corresponds to the same meaning.}
\label{tab:examples}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Table \ref{tab:examples} shows examples from the CoVoST 2 Es--En and the mTEDx Pt--En corpora.
In the first example, although both the baseline and the proposed method could predict \textit{a couple of beers} correctly, \textit{sure} or \textit{let me} were only predicted correctly by our model.
The translation generated by the proposed method also looks more natural.
In the second example, the proposed method correctly predicted \textit{only}, and also correctly translated \textit{queria}, the past tense of \textit{querer}, as \textit{wanted}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Discussion}

We discuss which layers are important in cross-lingual transfer in section \ref{sec:freeze},
the ASR pretraining language in section \ref{sec:investigate_s1},
the ST pretraining language pair in section \ref{sec:investigate_s2},
relation to the multilingual models in section \ref{sec:compare_multi},
and model sizes in section \ref{sec:model_size}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Ablation: Freezing Layers} \label{sec:freeze}

To investigate which layer of the Transformer contains important information to transfer, we froze some layers of the ST encoders and decoders in stage 3.
That is, for some of the layers at stage 3, the layers from the loaded checkpoint of stage 2 were used without updating the parameters.

CoVoST 2 was used for these experiments and the performance was investigated when French is used as the source language at stage 2.
We froze $6$ layers (1--6 or 7--12) of the encoder, all $12$ layers of the encoder, or all $6$ layers of the decoder.
We chose the best epoch in respect of the development set loss for evaluation.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% talbe 6
\begin{table}[b]
\input{13table06.tex}
\hangcaption{BLEU, chrF2, and TER scores of layer-freezing experiments on the CoVoST 2 test set. The ``Frozen layers'' column denotes the Transformer layers whose parameter was not updated during stage 3.}
\label{tab:covost2freeze}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We show the results when $\mathcal{S}_3$ is De--En and Es--En, which are the high-resource language pairs, in Table \ref{tab:covost2freeze}.
The first row shows the results of our proposed method (also shown in Table \ref{tab:covost2Cross}).
The rest shows the results of this experiment.
By comparing the results of freezing layers 1--6 or 7--12 of the encoders with those of freezing layers 1--12 of the encoders, it is observed that the BLEU scores decrease as the number of frozen layers increases.
Comparing the results of freezing layers 1--6 of the encoder with those of freezing layers 7--12 of the encoder, the scores decreased significantly when layers 7--12 were frozen.
That is, the parameters in the higher layers appear to be updated more than those in the lower layers during ST fine-tuning.
This suggests that the higher layers in the ST encoder are more language-specific, and lower layers are likely to contain more language-independent information.
Regarding the comparison between the encoder and decoder, the performance was mostly better when freezing layers 1--6 of the decoder than when freezing layers of the encoder.
This indicates that the encoder plays a more important role than the decoder when transferring the parameters between ST pretraining stage and ST fine-tuning stages.
Note that these statements hold only for many-to-one settings.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Using the ST Source Language for ASR Pretraining} \label{sec:investigate_s1}

In the experiments so far, English was used for ASR pretraining, which is different from the ST source language.
In an end-to-end ST task, only a pair of source speech and target text is required; however, usually, a triplet of source speech, source transcription, and target text are available.
Using different languages between the ASR and the ST tasks is a useful method when the ASR language is a high resource language and the transcriptions of the ST source language are not available \cite{bansal2019preasr}.
However, in most cases, ASR transcriptions are available for the ST source language, and it would be better to use the source language transcriptions when they are available.

Based on this assumption, we conducted experiments where the baseline is to use the ST source language as the ASR pretraining language.
For example, we first trained Fr ASR, and using the encoder of the trained model as the ST encoder, we trained Fr--En ST.
The proposed method here is to use an additional language pair at the ST fine-tuning stage.
For instance, we first trained Fr ASR, and trained Es--En at the ST pretraining stage, and then fine-tuned the model on Fr--En.

The mTEDx dataset was used for the experiments.
The settings were the same as those described in section \ref{mtedxconfig}, except that the ASR language was set to be the same as the ST source language in the ST fine-tuning stage.
We used Es--En as the language pair for ST pretraining because it is the highest-resource language.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 7
\begin{table}[b]
\input{13table07.tex}
\hangcaption{BLEU, chrF2, and TER scores of the experiments using the ST source language as the ASR language on mTEDx test set. The rows with ``B'' are the baseline results, and the rows with ``P'' are the results of the proposed method. $\dag$: The score is significantly superior to the other score ($p < 0.05$).}

\label{tab:investigate_s1_high}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The results are shown in Table \ref{tab:investigate_s1_high}.
Significance testing showed that the scores of our proposed method underperformed in Pt--En.
There were no significant differences in Fr--En and It--En.
We believe this is because of the problem known as catastrophic forgetting, where neural networks ``forget'' the information of the first task after the training of the second task.
That is, the information of the source language of $\mathcal{S}_3$ learned at stage 1 was likely to be lost during the training of stage 2.
It is concluded that the ASR pretraining method is better than our method when a sufficient amount of source language ASR data are available.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{What Language Pair to Choose at the ST Pretraining Stage}
\label{sec:investigate_s2}

We reported the results where we used Es--En as the ST pretraining language pair as the main results in section \ref{sec:main_results}.
Here, we conduct an extensive study to investigate the effect of using different language pairs at stage 2.

The mTEDx dataset was used for the experiments.
English was used as the ASR pretraining language.
Then, stage 2 was trained with four high-resource language pairs, Es--En, Pt--En, Fr--En, and It--En.
Finally, the models were fine-tuned on the three language pairs, which were not used in stage 2.
For all the experiments, the settings were the same as those described in section \ref{mtedxconfig}.

The results are presented in Table \ref{tab:investigate_s2_high}.
When $\mathcal{S}_3$ is Pt--En, Fr--En, or It--En, the best performance was obtained when Es--En was used as $\mathcal{S}_2$.
This is presumably because Es--En is the highest-resource language pair in the mTEDx dataset.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 8
\begin{table}[b]
\vspace{-0.5\Cvs}
\input{13table08.tex}
\caption{BLEU, chrF2, and TER scores using various language pairs at stage 2 on the mTEDx test set.}
\label{tab:investigate_s2_high}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

When $\mathcal{S}_3$ is Es--En, using Fr--En as $\mathcal{S}_2$ yielded the best performance.
Considering that the data size of Pt--En and Fr--En do not differ significantly (Table \ref{tab:dataset}), there seem to be factors other than the data size, which will be further investigated in future work.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Relation to the Multilingual Models}
\label{sec:compare_multi}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Multilingual Models and Application of the Proposed Method}

We have proposed to use an additional ST corpus at stage 2.
However, this may not be a fair comparison because we use an additional corpus.
Therefore, we conducted experiments with the multilingual training framework \cite{inaguma2019multilingual}.
After stage 1, we jointly trained the model using both language pairs used at stage 2 and stage 3.

First, a multilingual vocabulary was built with a size of $8,000$, constructed from all language pairs in the mTEDx dataset, following \cite{salesky2021mtedx}.
Then, using the En ASR model described in section \ref{mtedxconfig} to initialize the encoder, we trained multilingual models with two language pairs.
Language tags like \texttt{<es>} or \texttt{<fr>} were prepended to the target utterances so that they become the first tokens that the decoders generate.
We used validation sets of the two language pairs and validated the losses for each epoch.
When either of the validation sets met the condition described in section \ref{mtedxconfig}, the training was stopped.
During decoding, the first token (the language tag) was skipped.
Other settings were the same as the ones described in section \ref{mtedxconfig}.

As an extension of our cross-lingual transfer learning framework to this multilingual method, we also conducted experiments regarding the multilingual training as stage 2.
That is, we further fine-tuned the multilingual model using a single language pair at stage 3.
We used the same settings as the ones described in the previous paragraph.

The results are shown in Table \ref{tab:multi_baseline}.
The proposed method outperformed the multilingual baseline in almost all evaluations.
Comparing these results with those shown in Table \ref{tab:mtedxCross}, the results of the multilingual baseline outperformed those of the proposed method without multilingual ST pretraining.
It is concluded that when using the same data, although the multilingual method outperforms the proposed method without multilingual ST, the multilingual method can be applied to our method to further improve performance.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 9
\begin{table}[t]
\input{13table09.tex}
\hangcaption{BLEU, chrF2, and TER scores of the multilingual experiments on the mTEDx test set when En is used as $\bm{s}_1$. The rows with ``B'' on the first column show the results of the multilingual baseline without fine-tuning, and the rows with ``P'' show the results with fine-tuning. $\dag$: The score is significantly superior to the other score ($p < 0.05$).}
\label{tab:multi_baseline}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Using the ST Source Language for ASR Pretraining with Multilingual Models}

From what we have discussed in section \ref{sec:investigate_s1}, fine-tuning multilingual models using the same $\bm{s}_1$ as $\bm{s}_3$ would produce the best score.
Based on this assumption, using the pretrained ASR models described in section \ref{sec:investigate_s1}, we trained multilingual models using the translation target language pair and another language pair (e.g., Es--En and Pt--En), and then fine-tuned them using one language pair (e.g., Es--En).
We compare the performance of this method with the one without fine-tuning.

The results presented in Table \ref{tab:multi_s1s3} demonstrate that the proposed method performs well also in this setting.
Comparing Tables \ref{tab:multi_baseline} and \ref{tab:multi_s1s3}, using the same $\bm{s}_3$ as $\bm{s}_1$ is better than using En as $\bm{s}_1$ despite the difference in corpus size.
Significance tests were also performed comparing the corresponding ``Proposed'' lines of Table \ref{tab:multi_baseline} and Table \ref{tab:multi_s1s3}.
It was confirmed that the scores shown in Table \ref{tab:multi_s1s3} are superior to those in Table \ref{tab:multi_baseline}, except for the BLEU and TER scores of It--En where Es--En was also used for the multilingual training at stage 2 ($p < 0.05$).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 10
\begin{table}[t]
\input{13table10.tex}
\hangcaption{BLEU, chrF2, and TER scores of the multilingual experiments on the mTEDx test set, using the same $s_1$ as $s_3$. The rows with ``B'' on the first column show the results of the multilingual baseline without fine-tuning, and the rows with ``P'' show the results with fine-tuning. $\dag$: The score is significantly superior to the other score ($p < 0.05$).}
\label{tab:multi_s1s3}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Changing Model Size} \label{sec:model_size}

In the mTEDx experiments thus far, we have used models with $6$ encoder layers and $3$ decoder layers following \cite{salesky2021mtedx}.
To investigate how changing the model size affects the performance of our method, we conducted experiments using models with $12$ encoder layers and $6$ decoder layers, which is the same setting as the one used for the CoVoST 2 dataset.

The results are presented in Table \ref{tab:model_size}.
Increasing the model size significantly improved the ST performance when high-resource language pairs were used.
Our method outperformed the baseline also in this setting, which means that the method can work with different model sizes.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 11
\begin{table}[t]
\input{13table11.tex}
\caption{Results of the experiments changing model size on the mTEDx test set.}
\label{tab:model_size}
\vspace{-0.5\Cvs}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}

We proposed a method for end-to-end ST with cross-lingual transfer learning, and showed that our method is effective in many-to-one settings when English is used as the ASR pretraining language.
Through an ablation study investigating which Transformer layers contain important information to transfer, it was demonstrated that the lower layers of the encoder are likely to contain language-independent information for cross-lingual transfer.
Extensive studies were conducted on (1) ASR pretraining language, (2) ST pretraining language pair, (3) multilingual methods, and (4) model sizes.
It was demonstrated that
(1) Using the same language as the ASR pretraining language and the ST fine-tuning source language results in good performance.
(2) A high-resource language pair is a good choice for the ST pretraining language pair.
(3) The proposed method works well when used in conjunction with multilingual methods.
(4) The proposed method can operate with different model sizes.
In the future, the effectiveness of cross-lingual transfer learning will be explored in other paradigms for multilingual ST, namely one-to-many and many-to-many ST.


\vspace{-0.5\Cvs}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment

This work was supported by Grant-in-Aid for Young Scientists \#19K20343, JSPS.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.6}
\bibliography{13refs}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 12
\begin{table}[b]
\input{13table12.tex}
\caption{The number of utterances of the CoVoST 2 English ASR data used in the experiments.}
\label{tab:covost2_asr_stat}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 13
\begin{table}[b]
\input{13table13.tex}
\hangcaption{WER of the ASR experiments on the CoVoST 2 test set. The first row displays the result of an ASR experiment using Transformer with $12$ encoder layers and $6$ decoder layers, for the pretraining of CoVoST 2 ST experiments. The second row is for the pretraining of mTEDx ST experiments, where Transformer with $6$ encoder layers and $3$ decoder layers are used.}
\label{tab:covost2_asr_results}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 14
\begin{table}[b]
\input{13table14.tex}
\caption{The number of utterances of the mTEDx ASR data used in the experiments in section \ref{sec:investigate_s1}.}
\label{tab:mtedx_asr_stat}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 15
\begin{table}[b]
\input{13table15.tex}
\caption{WER of the ASR experiments (described in section \ref{sec:investigate_s1}) on the mTEDx test set.}
\label{tab:mtedx_asr_results}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\appendix

\section{ASR Statistics and Results} \label{sec:appendixA}
Here, the statistics of the ASR datasets used for ASR pretraining and the results of the ASR experiments are reported.
Table \ref{tab:covost2_asr_stat} lists the statistics for the CoVoST 2 English ASR data.
Table \ref{tab:covost2_asr_results} lists the results of the ASR experiments on the CoVoST 2 dataset reported in word error rate (WER).
Table \ref{tab:mtedx_asr_stat} lists the statistics of the mTEDx ASR data used in the experiments.
Table \ref{tab:mtedx_asr_results} lists the results of the ASR experiments on the mTEDx dataset reported in WER.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Data Statistics and Experimental Results of Low-resource Language Pairs}
\label{sec:appendixB}

In the main text, the statistics and results of high-resource language pairs were presented.
Here, the statistics of low-resource language pairs are presented.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 16
\begin{table}[b]
\input{13table16.tex}
\hangcaption{Number of utterances for each language pair in CoVoST 2 and mTEDx datasets used in our ST experiments (low-resource language pairs). The languages shown are the source languages and English is the target language.}
\label{tab:dataset_low}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 17
\begin{table}[b]
\input{13table17.tex}
\hangcaption{BLUE scores of cross-lingual ST experiments on the CoVoST 2 test set. The baseline uses En as $\bm{s}_1$, Fr as $\bm{s}_2$, and the languages shown as $\bm{s}_3$. English was used as the target language.}
\label{tab:covost2Cross_low}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Datasets}

Table \ref{tab:dataset_low} shows the dataset statistics of the low-resource language pairs in the CoVoST 2 and mTEDx datasets.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Results}

Table \ref{tab:covost2Cross_low} shows the ST experiment results for CoVoST 2 low-resource language pairs.
\pagebreak
Table \ref{tab:mtedxCross_low} shows the results for mTEDx low-resource language pairs.
The scores are random, and it is not easy to draw useful conclusions from these results.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 18
\begin{table}[t]
\input{13table18.tex}
\hangcaption{BLEU scores of cross-lingual ST experiments on the mTEDx test set. The baseline uses En as $\bm{s}_1$, Es as $\bm{s}_2$, and the languages shown as $\bm{s}_3$. English was used as the target language. $\dag$:~The score is significantly superior to the other score ($p < 0.05$).}
\label{tab:mtedxCross_low}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Shuichiro Shimizu}{%
Shuichiro Shimizu received his B.S. in Electrical and Electronic Engineering from Kyoto University in 2021.
His research interests are natural language processing, in particular speech translation.
}

\bioauthor[:]{Chenhui Chu}{%
Chenhui Chu received his B.S. in software engineering from 
Chongqing University in 2008, and his M.S. and Ph.D. in Informatics 
from Kyoto University, in 2012 and 2015, respectively. He is currently 
a program-specific associate professor at Kyoto University. His 
research interests include natural language processing, specifically 
machine translation and multimodal machine learning. 
}

\bioauthor[:]{Sheng Li}{%
Sheng Li received his B.S. and M.E. degrees in 2006 and 2009 from Nanjing University, Nanjing, China, and his Ph.D. degree from Kyoto University, Kyoto, Japan, in 2016. He is currently with the National Institute of Information and Communications Technology (NICT), Kyoto, Japan, as a researcher working on speech recognition. He is a member of the International Speech Communication Association (ISCA) and IEEE. 
}

\bioauthor[:]{Sadao Kurohashi}{%
Sadao Kurohashi received the B.S., M.S., and Ph.D. degrees in Electrical Engineering from Kyoto University in 1989, 1991, and 1994, respectively.
He was a visiting researcher of IRCS at the University of Pennsylvania, in 1994.
He is currently a professor of the Graduate School of Informatics of Kyoto University.
His research interests include natural language processing and knowledge acquisition/representation.
}

\end{biography}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\biodate



\end{document}

