\documentclass[english]{jnlp_3.8}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{array}
\usepackage{hangcaption_jnlp}
\allowdisplaybreaks


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
%%%%\usepackage{subcaption}
%%%%\usepackage{textcomp}
%%%%\usepackage{color}
%%%%\usepackage{enumerate}
\usepackage{booktabs}
%%%%\usepackage{microtype}
%%%%\usepackage{ragged2e}
%%%%\usepackage{array}
%%%%\usepackage{makecell} % Optional, for more control over cell formatting
%%%%\usepackage{graphicx} % Needed for \resizebox, which can be useful in some cases
%%%%\newcommand*{\red}{\textcolor{black}}
%%%%\newcommand*{\redd}{\textcolor{black}}
%%%%\newcommand*{\reddd}{\textcolor{black}}
%%%%\newcommand*{\redddd}{\textcolor{black}} % v6
%%%%\newcommand*{\blue}{\textcolor{black}}
%%%%\newcommand*{\pink}{\textcolor{black}} % v6
%%%%\newcommand*{\cyan}{\textcolor{black}}
%%%%\newcommand*{\green}{\textcolor{black}}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\Volume{32}
\Number{2}
\Month{June}
\Year{2025}
%%%%\Issuetitle{「　　　　」特集号}

\received{2024}{8}{6}
\revised{2024}{12}{1}
\accepted{2025}{1}{15}

\setcounter{page}{404}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{End-to-end Simultaneous Speech Translation with Style Tags using Human Simultaneous Interpretation Data}
\eauthor{Yuka Ko\affiref{Author_1} \and Ryo Fukuda\affiref{Author_1} \and Yuta Nishikawa\affiref{Author_1} \and Yasumasa Kano\affiref{Author_1} \and \\ Katsuhito Sudoh\affiref{Author_1}\affiref{Author_2} \and Sakriani Sakti\affiref{Author_1} \and Satoshi Nakamura\affiref{Author_1}\affiref{Author_3}} 
\eabstract{%
Simultaneous speech translation (SimulST) translates speech incrementally, requiring a monotonic input-output correspondence to reduce latency. This is particularly challenging for distant language pairs, such as English and Japanese, as most SimulST models are trained using offline speech translation (ST) data, where the entire speech input is observed during translation. In simultaneous interpretation (SI), a simultaneous interpreter translates source language speech into target language speech without waiting for the speaker to finish speaking. Therefore, the SimulST model can learn SI-style translations using SI data. However, owing to the limited availability of SI data, fine-tuning an offline ST model using SI data may result in overfitting. To address this problem, we propose an efficient training method for the speech-to-text SimulST model using a combination of small SI and relatively large offline ST data. We trained a single model with mixed data by incorporating style tags to instruct the model to generate either SI or offline-style outputs. This approach, called \emph{mixed fine-tuning with style tags}, can be extended further using the \emph{multistage self-training} approach. In this case, we use the trained model to generate pseudo-SI data. Our experimental results for several test sets demonstrated that our models trained using mixed fine-tuning and multistage self-training outperformed baselines across various latency ranges. 
}
\ekeywords{Simultaneous Speech Translation, Simultaneous Interpretation, Domain \linebreak Adaptation}

\headauthor{Ko et al.}
\headtitle{End-to-end Simultaneous Speech Translation with Style Tags using Human SI Data}

\affilabel{Author_1}{}{Nara Institute of Science and Technology}
\affilabel{Author_2}{}{Nara Women's University}
\affilabel{Author_3}{}{The Chinese University of Hong Kong, Shenzhen}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

\renewcommand{\thefootnote}{}
\footnote[0]{Part of this work was presented at the 20th International Conference on Spoken Language Translation \cite{ko-etal-2023-tagged}.}
\renewcommand{\thefootnote}{\arabic{footnote}}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}

Simultaneous speech translation (SimulST) incrementally translates speech without waiting for a sentence to end. 
As SimulST aims for a monotonic translation process to keep up with the input speech, it is essential to maintain the original word order of the source language. 
However, such monotonic translation is challenging for distant language pairs with different word orders, such as English and Japanese.\footnote{This is because English is an SVO language, while Japanese is an SOV language.} 
This is because most recent SimulST studies have relied solely on parallel corpora with offline translations.
Offline speech translation (ST) prioritizes fluency and accuracy over speed, resulting in the long-distance reordering of source words. 
Consequently, training SimulST systems with offline ST data leads to difficulties in producing monotonic translations. 
By contrast, simultaneous human interpreters prioritize generating translations as quickly as possible while maintaining monotonicity between the source and target content. 
By utilizing sentence-level human simultaneous interpretation (SI) data, in which the source and target contents correspond monotonically, it is possible to train SimulST models to mimic the monotonic translation style of human interpreters. 
However, recent SimulST systems cannot learn such SI-like translations because they use offline ST data. 

To address this problem, SI data is used to train a SimulST model to effectively mimic SI.
Over the past few decades, several English-Japanese SI corpora have been developed \cite{Tohyama-et-al-2004,shimizu-etal-2013-constructing,matsushita-et-al-2020,doi-etal-2021-large}. 
Despite these efforts, the amount of SI data remains significantly smaller than that of bilingual data based on offline translations.
Fine-tuning an offline ST model with limited SI data can lead to overfitting. 
Another approach involves training the model using offline and SI data to address data scarcity. 
However, a simple mixture of data can confuse the different output styles between the offline ST and SI. 

In this paper, we propose a method for training a SimulST model in a speech-to-text setting using a combination of SI and offline ST data that incorporate style tags to instruct the model to generate either SI-style or offline-style outputs.
We call this primary approach \emph{mixed fine-tuning with style tags}. 
The proposed method enables SI-style output using SI-style tags during decoding. 
In addition, leveraging the rich linguistic patterns and fluent translation capabilities learned from large-scale offline ST data helps the model differentiate between two output styles when guided by appropriate style tags.
This approach can be further extended by a \emph{multistage self-training} approach using a trained model to generate pseudo-SI data. We used pseudo-SI data to fine-tune the trained model, and this process was repeated. 
The motivation for this approach is to alleviate SI data scarcity using pseudo-SI data. 

Experimental results demonstrated that our \emph{mixed fine-tuning with style tags} approach improves the BLEURT \cite{sellam-etal-2020-bleurt} and COMET-QE \cite{rei2020comet,rei-etal-2021-references} scores compared to the baselines across various latency ranges in three English-Japanese test sets: (1) offline test of tst-COMMON in MuST-C v2 \cite{di-gangi-etal-2019-must}, (2) SI reference test of NAIST-SIC-Aligned-ST \cite{ko-etal-2023-tagged}, and (3) chunk-wise monotonic translation (CMT) reference test \cite{fukuda2024cwmt}.
We also observed further improvements when the proposed \emph{multistage self-training} approach was applied. 
Further analyses showed that the proposed models produced more SI-style outputs than baseline models in terms of semantic sentence similarity in various test sets.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Work}

Over the past few decades, several studies have been conducted on the simultaneous translation of both text and speech  \cite{fugen2007simultaneous,oda2014optimizing,dalvi-etal-2018-incremental}.
The most recent approaches are based on deep neural networks and have evolved with advancements in neural machine translation (NMT) \cite{gu-etal-2017-learning} and neural automatic speech recognition  \cite{rao2017exploring}.
A key advantage of neural SimulST methods \cite{ma-etal-2020-simulmt,ren-etal-2020-simulspeech} is their end-to-end modeling of the entire process, which enhances the efficiency compared to cascade approaches. 
These end-to-end SimulST models are typically trained using offline ST corpora, such as MuST-C \cite{di-gangi-etal-2019-must}, which comprises subtitles from TED talks.\footnote{\url{http://www.ted.com}} 
When training on offline data, the model was set up to perform full-sentence translation, where it waits until the entire speech input is received before starting the translation process.
Offline translation tends to be natural and fluent, but is not translated with less latency.
Although offline ST data are not designed for learning SimulST, they have been mainly used for training SimulST because large-scale offline data can be easily prepared.

For the English-Japanese language pair, efforts have been made to develop SI corpora \cite{Tohyama-et-al-2004,shimizu-etal-2013-constructing,matsushita-et-al-2020,doi-etal-2021-large}. 
However, the number of SI corpora is limited because of the challenges in developing high-quality SI data. 
In this study, we address the shortage of SI data by utilizing not only SI data but also large-scale offline translation data.
One possible method is to transfer the translation skills learned from a model trained on large offline ST data (for example, vocabulary, syntactic structures, and common linguistic patterns) to the SimulST task.
This can be considered as a domain adaptation from resource-rich offline translations to simultaneous resource-poor translations. 
In typical domain adaptation, an out-of-domain model is fine-tuned with in-domain data \cite{luong-manning-2015-stanford,sennrich-etal-2016-controlling}; however, this often leads to overfitting owing to the small size of the in-domain data \cite{chu-etal-2017-empirical}.
An alternative approach is the tag-based NMT, which has been used to control translation politeness \cite{sennrich-etal-2016-controlling} and enable zero-shot multilingual NMT \cite{johnson-etal-2017-googles}. 
This tag-based method has been extended to multidomain fine-tuning \cite{kobus-etal-2017-domain} and mixed fine-tuning \cite{chu-etal-2017-empirical}, where NMT models are fine-tuned using a mix of in-domain and out-of-domain data.
Tagged back translation  \cite{caswell-etal-2019-tagged} applies this approach to back-translation-based data augmentation, distinguishing between source-language sentences from parallel corpora and those with potential noise obtained from back-translated data. 
Our study was inspired by tag-based domain-adaptation methods to address the scarcity of SI data. 
For domain adaptation from resource-rich offline translation to resource-poor simultaneous translation while mitigating overfitting for SI data, 
we focused on controlling style differences between SI-style and offline-style outputs using tagging approaches. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Differences between Offline Translation and Simultaneous Interpretation}

Because SI is a cognitively demanding task, simultaneous human interpreters employ strategies such as segmentation, summarization, and generalization \cite{he-etal-2016-interpretese}. In practice, they often follow the speaker's speech using summarizations or omissions, and maintaining the word order of the source language is a crucial strategy to minimize delays and alleviate cognitive load. 
This approach is particularly important for distant language pairs with different word orders such as English and Japanese \cite{mizuno2017simultaneous}. 
These SI-specific strategies (for example, summarization, omission, and strict maintenance of word order) result in SI outputs that are stylistically different from the offline translation outputs, thereby highlighting the significant style differences between SI and offline translation.

Figure~\ref{tab:example-sentence-in-si-off} illustrates this difference using an example of a Japanese offline translation and an SI transcript for a given English source sentence. 
The solid lines in the figure indicate word correspondence.
From this figure, the following can be observed:
\begin{itemize}
    \item Most English words are translated into Japanese in the offline translation, whereas some of these words are missing in the SI transcript. 
    \item The SI attempts to translate the first half of the input earlier than the second half, resulting in some unnaturalness. 
    By contrast, offline translation maintains naturalness in Japanese by reordering the input English over longer distances. 
\end{itemize}
These observations highlight the key differences between offline translation and SI. 
SI prioritizes simultaneity in delivering content as quickly as possible and manage the interpreter's working memory. 
Therefore, it is crucial to use SI data to train the SimulST model to enhance its simultaneity.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1
\begin{figure}[b]
\begin{center}
\includegraphics{32-2ia1f1.eps}
\end{center}
\hangcaption{Example of English-to-Japanese offline translation and SI. The example is from NAIST-SIC-Aligned \protect\cite{zhao-etal-2024-naist}.}
\label{tab:example-sentence-in-si-off}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 2
\begin{figure}[b]
\begin{center}
\includegraphics{32-2ia1f2.eps}
\end{center}
\caption{Mixed fine-tuning with style tags}
\label{fig:mixed-style}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Proposed Method}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Mixed Fine-tuning with Style Tags}
\label{method}

We propose a method that utilizes a relatively large offline translation corpus to mitigate the lack of SI data for training the SimulST model. 
Figure~\ref{fig:mixed-style} presents an overview of \emph{mixed fine-tuning with style tags}.
Using this method, we train a single model using mixed offline and SI data. 
In addition, when training the SimulST model, we incorporated style tags to instruct the model to generate either SI or offline-style outputs. 
The proposed method enables the distinction of both output styles with style tags while utilizing the knowledge of large amounts of offline data for both style outputs.
It was inspired by tag-based domain-adaptation methods. 
This can be considered as domain adaptation from a resource-rich offline translation to a resource-poor SI. 

According to a previous study by \citeA{chu-etal-2017-empirical}, fine-tuning an out-of-domain model with a small amount of in-domain data often leads to overfitting. 
They addressed this problem by proposing a mixed training approach that leveraged both small in-domain data and large out-of-domain data using in-domain and out-of-domain tags. 
Our study was inspired by previous tag-based approaches that aimed to address the scarcity of SI data.
Using the proposed approach, we attempted to alleviate the problem of overfitting the SimulST model to small amounts of SI data. 
In the proposed approach, we fine-tune the pretrained ST model using a mix of in-domain and out-of-domain offline ST data, incorporating style tags to instruct the model to produce either SI-style or offline-style outputs. 
This approach allows the model to leverage a large amount of out-of-domain data to mitigate overfitting while maintaining the ability to generate both styles of output. 
By contrast, the conventional SI fine-tuning (SI FT) approach fine-tunes the pretrained ST model using only in-domain SI data without style tags. Although straightforward, this method risks overfitting owing to the limited size of the in-domain data.
In the implementation, we used style tags at the beginning of the target strings during training and enforced the prediction of a specified tag during inference. We use \texttt{<si>} tag for SI and \texttt{<off>} tag for offline translation.
Assuming an SI transcript written in Japanese: 私は、買った。ペンを、 and an Offline transcript: 私はペンを買いました for the English input: \emph{I bought a pen.} as a training example;
we placed the SI-style tag at the beginning of the SI transcript, as shown in the following sentence: 
\begin{quote}
\texttt{<si>}私は、買った。ペンを、(``I'' ``bought'' ``a pen'')
\end{quote}

By contrast, we placed the offline-style tag at the beginning of the offline transcript in the following sentence: 
\begin{quote}
\texttt{<off>}私はペンを買いました
\end{quote}


In the training step, we fine-tuned the pretrained ST model by training SimulST using mixed data with style tags. 
In the inference step, forced decoding was applied using prefix tags. 
When SI-style outputs were obtained, we placed \texttt{<si>} tags, and when offline-style outputs were obtained, we placed \texttt{<off>} tags. 
The proposed method simplifies the process by generating multiple-style outputs using a single model by applying prefix-style tags. 
This is significantly different from the previous straightforward approach, which fine-tuned each pretrained model directly using the data in each style.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Multistage Self-training in Mixed Fine-tuning with Style Tags}
\label{proposed_method:multistage_selftrain}

\emph{Mixed fine-tuning with style tags} in Section \ref{method} uses only the original SI data.
The fact that the portion of the original SI data is small compared to the offline data is a concern.
We believe that increasing the SI data portion by training \emph{mixed fine-tuning with style tags} using pseudo-SI data will further alleviate the small SI data problem. 

Data augmentation based on generated synthetic data is one of the main approaches \citeA{zhang-zong-2016-exploiting} proposed to fully use the source language monolingual data for NMT by self-learning with synthetic data generated from the initial NMT system and multi-task learning using two NMTs. 
Self-training \cite{zoph2020rethinking} can improve a model by generating synthetic pseudo-labeled data from unlabeled data using an initial model when labeled data are limited. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 3
\begin{figure}[b]
\begin{center}
\includegraphics{32-2ia1f3.eps}
\end{center}
\caption{Multistage self-training in mixed fine-tuning with style tags}
\label{fig:self-training-mixed-style}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Motivated by these studies, we propose combining a self-training method, in which pseudo-generated SI data are used to learn in multistage, with the mixed fine-tuning above.
Figure~\ref{fig:self-training-mixed-style} presents an overview of the use of self-training in mixed fine tuning using style tags. 
The stage of multistage self-training is denoted by $N$.
In the $N=2$ stage, using a pretrained SimulST, we generate pseudo-SI data by applying \texttt{<si>} tags to source speech inputs in offline ST training data.
Then, we fine-tune the pretrained SimulST with the generated pseudo-SI data with \texttt{<si>} tags with original SI and offline data in our \emph{mixed fine-tuning with style tags} approach. 
Because we used source speech inputs from offline ST training data to generate pseudo-SI data, the amount of pseudo-SI data was as large as that of the offline ST data. The pseudo-SI data were used for training with \emph{mixed fine-tuning with style tags} in addition to the original SI and offline ST data.
We refer to the newly trained model as $\text{Style-MultiAug-2}$ FT in Figure~\ref{fig:self-training-mixed-style}. 
This operation continues until $N_{\text{max}}$ and the final $\text{Style-MultiAug-}N_{\text{max}}$ FT model is adopted as the proposed model.  
Thus, pseudo-SI data can be generated using the models already trained in the previous step. 
Using this approach, we expect that when $N$ increases, SimulST is trained using more diverse pseudo-SI data. 
In this method, the SimulST model was trained using only self-generated data from previously trained models. 
Therefore, the preparation of additional ST data or external resources for preparing the pseudo-SI data are not required. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experimental Setup}

The following experiments investigated the effectiveness of the \emph{mixed fine-tuning with style tags} and \emph{multistage self-training} approaches with several test data in full-sentence offline ST and SimulST tasks, compared to baseline approaches without style tags. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[b]
\input{01table01.tex}%
\caption{Data sizes of offline, SI, and CMT data for fine-tuning. }
\label{tab:data_size}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Dataset}

For the SI training data, we used NAIST-SIC-Aligned-ST \cite{ko-etal-2023-tagged}.\footnote{\url{https://dsc-nlp.naist.jp/data/NAIST-SIC/Aligned-ST}} 
The SI data were English-Japanese speech-to-text data, which were created by automatic alignment between the parallel text segments from NAIST-SIC-Aligned \cite{zhao-etal-2024-naist} and the corresponding audio tracks in MuST-C.
The NAIST-SIC alignment was constructed by applying automatic sentence alignment using NAIST-SIC\footnote{\url{https://dsc-nlp.naist.jp/data/NAIST-SIC/2022}} \cite{doi-etal-2021-large}. 
For NAIST-SIC-Aligned, we selected INTRA and AUTO-DEV portions as the training and development data, respectively.
Segments that did not align with the source speech were excluded from the aligned dataset.
Table~\ref{tab:data_size} shows the size of all data.
For the offline training data, we used MuST-C \cite{di-gangi-etal-2019-must} v2 English-Japanese data as our offline speech translation corpus. 
The following three test sets were used for evaluation:



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Offline test}

The tst-COMMON portion of MuST-C \cite{di-gangi-etal-2019-must} was used as the offline test set. 
For the offline data, the source speech content was maintained in the target reference without omissions. 
While naturalness and fluency are considered in offline ST, latency is not taken into account.
In Japanese, the distances between the source words and the corresponding target words are generally large. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{SI reference test}

We used the test set in NAIST-SIC-Aligned-ST \cite{ko-etal-2023-tagged}, which is based on the AUTO-TEST portion of NAIST-SIC-Aligned \cite{zhao-etal-2024-naist}. 
We can evaluate model performance using SI sentences, which include omissions and reordering from real professional human simultaneous interpretations. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{Chunk-wise Monotonic Translation (CMT) reference test}

We also used the NAIST English-to-Japanese CMT reference dataset\footnote{\url{https://dsc-nlp.naist.jp/data/NAIST-SIC/Aligned-Chunk_Mono-EJ}} to evaluate the SimulST models. 
The CMT reference test data were developed by \citeA{fukuda2024cwmt} to isolate the effects of word order differences from translation omissions, a factor that often arises in simultaneous interpretation owing to summarization and generalization strategies. 
Both appropriate omissions and omissions resulting from interpreter errors are present in actual SI data, making it challenging to distinguish between them.
\citeA{fukuda2024cwmt} also highlighted the difficulty for SI models to learn  less critical phrases, advocating for SI models that focus solely on maintaining the source word order. In a related study, \citeA{doi-etal-2024-word} analyzed the characteristics of CMT sentences and identified grammatical structures that complicate monotonic translations in English-Japanese SI, suggesting that conventional SI reference test sets may underestimate model performance. Both studies supported the use of a monotonic translation-based dataset for a more reliable evaluation of SimulST models.
Accordingly, we employed the CMT reference test set to assess the ability of SimulST to maintain the word order and preserve the speaker's original content without the confounding effects of summarization or omission.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Offline Speech Translation}

Recently, one of the main approaches to SimulST has been to utilize a large-scale pretrained offline ST model with an online simultaneous decoding policy. 
\citeA{tsiamas-etal-2022-pretrained} constructed an end-to-end multilingual offline ST model based on large pretrained speech and text models to reduce the required amount of ST data. 
They used wav2vec 2.0 or Hidden-Unit BERT (HuBERT) \cite{hsu2021hubert} as a speech encoder, and mBART50 \cite{liu-etal-2020-multilingual-denoising,tang2021multilingual} as a text decoder. 
Both wav2vec 2.0 and HuBERT are based on the same transformer speech encoder architecture and are pretrained using self-supervised learning. They directly take the raw speech waveform as input, from which they first extract low-level acoustic features and then process these features to generate contextualized representations.
Wav2vec 2.0, was trained to identify the true speech representation from a masked time step by solving a contrastive task on quantized representations. 
HuBERT predicts the masked time steps by computing the loss against pseudo-labels obtained from interactive offline clustering. 
mBART50 is an encoder-decoder transformer-based language model that is trained to reconstruct a sequence from its noisy version and is later extended to a multilingual version. 
The best ST system in \citeA{tsiamas-etal-2022-pretrained} comprises HuBERT as a speech encoder and mBART50 as a text decoder. 
ST models pretrained using self-supervised learning can achieve highly competitive results even when a limited amount of labeled data is available. 
\citeA{liu2020lowlatency} proposed the local agreement (LA) approach as an online simultaneous decoding policy that outputs the agreeing prefixes of the two consecutive chunks.
\citeA{polak-etal-2022-cuni} applied the LA decoding policy for an offline ST model based on pretrained wav2vec 2.0 and mBART50, resulting in the best-scored system in the IWSLT 2022 evaluation campaign.

Following these previous studies, our system is based on offline ST models using pretrained HuBERT \cite{hsu2021hubert} and mBART50 \cite{liu-etal-2020-multilingual-denoising,tang2021multilingual}. 
First, we constructed an initial offline ST model by connecting these two pretrained models, as described in Section \ref{initlal_offline_st_model}. 
We then fine-tuned the initial models using SI data in the baseline and proposed approaches, as described in Sections \ref{simulst_baseline} and \ref{simulst_proposed}. 
Finally, we applied the LA simultaneous decoding policy to the trained ST models.  


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Initial offline ST model before fine-tuning with SI and offline data}
\label{initlal_offline_st_model}

Our ST models used the pretrained HuBERT \cite{hsu2021hubert} and mBART50 \cite{liu-etal-2020-multilingual-denoising,tang2021multilingual}. 
Our SimulST system was implemented using \texttt{fairseq} \cite{ott-etal-2019-fairseq} based on the code available in \citeA{tsiamas-etal-2022-pretrained}.\footnote{\url{https://github.com/mt-upc/iwslt-2022}} 
We used 250,000 unigram vocabularies with SentencePiece \cite{kudo-richardson-2018-sentencepiece} attached to mBART50. 
These two models were connected using an interconnection \cite{2023-inter-connection} that assigned weights to each transformer layer of the encoder and integrated the output tensors of each layer through a weighted sum and length adapter \cite{tsiamas-etal-2022-pretrained}.
The length adapter was a three-layer convolutional network with 1,024 channels, a stride of two, and a GELU activation function. 
The inputs were waveforms sampled with a 16-kHz sampling rate normalized to zero mean and unit variance.
During training, we used WavAugment \cite{wavaugment2020} with a probability of 0.8 for augmenting source audio. 
The model was trained on the MuST-C \cite{di-gangi-etal-2019-must}, CoVoST-2 \cite{wang-etal-2020-covost}, Europarl-ST \cite{Europarl2020}, and TED-LIUM \cite{Rousseau2012TEDLIUMAA} datasets.
Gradient accumulation was applied with 24 steps and data-parallel computations to achieve a batch size of approximately 24 million tokens. 
The model was optimized using Adam, with $\beta_1=0.99$ and $\beta_2=0.98$.
We set the learning rate to $2.5\times10^{-4}$ and saved the models every 1,000 updates. 
The learning rate was managed by a tri-stage scheduler, with warm-up, hold, and decay phrases occupying 0.15, 0.15, and 0.70 of the training time.
The initial and final learning rates were scaled to 0.01. 
Sentence averaging and gradient clipping were set to 20. 
A dropout rate of 0.1 was applied before every nonfrozen layer. 
The output of the encoder feature extractor was augmented with time masking, where the spans of ten frames were masked with a probability of 0.2, and channel masking, where the spans of 20 channels were masked with a probability of 0.1.   
We used cross-entropy loss with label smoothing and a 20\% probability mass. 
We adopted checkpoints by averaging over five of the best checkpoints according to the loss in the development set. 

We used this trained model for fine-tuning with the SI and offline data.\footnote{The initial offline ST model is the same as the \emph{base} model in the following paper for the NAIST IWSLT 2023 simultaneous speech-to-speech model for simultaneous ST tasks \cite{fukuda-etal-2023-naist}.} 
During fine-tuning, we set the learning rate to $2.5\times10^{-5}$, saved the models every 800 updates, and selected the best checkpoint based on the loss in the development set. The training process was terminated if the development loss did not decrease for four consecutive updates.
We applied gradient accumulation with 16 steps and data-parallel computations to achieve a batch size of approximately four million tokens. 
Following \citeA{tsiamas-etal-2022-pretrained}, to avoid overfitting the small SI data, the parameters of the following components were fixed: the feature extractor and feed-forward layers of the encoder, and the embedding, self-attention, and feed-forward layers of the decoder.\footnote{In \citeA{fukuda-etal-2023-naist} and \citeA{ko-etal-2023-tagged}, the models were trained with bilingual prefix alignment pairs extracted by a prefix alignment approach \cite{kano-etal-2022-simultaneous}. When applying the prefix-alignment approach, we extracted prefix-to-prefix translation pairs from the available training sets, which could be obtained as intermediate translation results using a given offline translation model.However, we did not apply the prefix alignment approach in this study, because the overall scores were better without bilingual prefix pairs.}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Simultaneous Speech Translation}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Baseline models}
\label{simulst_baseline}

The baseline models without style tags were as follows: 
    \begin{description}
        \item[Offline FT] Fine-tuned using the offline data.
        \item[SI FT] Fine-tuned using the SI data.
        \item[Mixed FT] Fine-tuned using both the offline and SI data.
    \end{description}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Proposed models}
\label{simulst_proposed}

The proposed models with style tags are as follows. 
\begin{description}
    \item[Style FT] Fine-tuned using both the offline and SI data with the style tags.
    \item[$\text{Style-MultiAug-$N_{max}$}$ FT] Multistage self-training was applied to mixed fine-tuning with style tags approach. In this experiment, multiple fine-tuning stages were repeated until convergence was achieved using the development data at each stage. As shown in Table \ref{tab:validation_loss_multift}, we used the best model from the fourth stage, which results in the lowest validation loss from $N=1$ to $N=5$. Hereafter, we refer to the proposed model created by this method as Style-MultiAug-4 FT, where $N_{max}=4$ in Style-MultiAug-$N_{max}$.\footnote{To train the first $\text{Style-MultiAug-1}$ initially, we used the trained Style FT model for generating pseudo-SI data and applied it to the initial offline ST model in Section~\ref{initlal_offline_st_model}.}
\end{description}

In this work, the style tags, that is, \texttt{<si>} and \texttt{<off>} (see Section \ref{method}) are not registered in the subword vocabulary attached to mBART50 model as special symbols, and thus, they should be tokenized such as ``\_\texttt{<}\textvisiblespace\texttt{si}\textvisiblespace\texttt{>}'' and ``\_\texttt{<}\textvisiblespace\texttt{off}\textvisiblespace\texttt{>}''.
Here, ``\_'' is the meta-character representing white spaces in an original string by SentencePiece \cite{kudo-richardson-2018-sentencepiece}, and ``\textvisiblespace'' represents a white space in a tokenized string.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 2
\begin{table}[b]
\input{01table02.tex}%
\caption{Quality metrics used in our experiments}
\label{tab:quality_metrics_lists}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[b]
\input{01table03.tex}%
\caption{Each minimum development loss of Style-MultiAug-$N$ FT from $N=1$ to $N=5$.}
\label{tab:validation_loss_multift}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Evaluation Metrics}

The SimulST systems were evaluated using SimulEval\footnote{\url{https://github.com/facebookresearch/SimulEval}} \cite{ma-etal-2020-simuleval}, which automatically performs simultaneous decoding given a policy and reports several popular latency metrics.
As a SimulST decoding policy, we applied LA (LA-$n$) \cite{liu2020lowlatency,polak-etal-2022-cuni} that displays agreeing prefixes of $n$ consecutive chunks.  
We set $n=2$ for the local agreement (LA-2), because \citeA{polak-etal-2022-cuni} reported that it achieves the best latency-quality trade-off among various LA-$n$ strategies. 
Speech segments were evaluated at intervals of \{200, 400, 600, 800, 1,000\} ms to control for the quality-latency trade-off. 
Hypotheses for speech segments were generated using a beam search with a beam size of five. 
The translation quality was evaluated using BLEU \cite{papineni-etal-2002-bleu}, BLEURT \cite{sellam-etal-2020-bleurt} and COMET-QE \cite{rei2020comet,rei-etal-2021-references}. 
BLEU was computed directly using SimulEval, which uses SacreBLEU \cite{post-2018-call}.
The BLEURT and COMET-QE scores were evaluated separately using the translations generated after all the sentences were generated using SimulEval. 
Table~\ref{tab:quality_metrics_lists} lists the quality metrics used in our experiments. 
Because SI is a cognitively demanding task and human simultaneous interpreters try to summarize or generalize the contents considering the meaning of the speaker's speech, in this study, we focused on scores based on semantic similarity measures, such as BLEURT and COMET-QE. 
They are based on the embedding of a sentence rather than BLEU, which is a traditional quality metric for machine translation evaluated on textual surface agreements. 
Compared to BLEURT and BLEU, which are reference-based evaluation metrics, we used reference-less COMET-QE to directly measure the amount of the speaker's speech contents included in the test translation. 

The latency in SimulST was evaluated using the average token delay (ATD) \cite{kano2024atd}, as implemented in SimulEval. 
ATD is a latency evaluation metric that focuses on the end timings of partial translations in simultaneous translations.\footnote{Although average lagging \cite{ma-etal-2019-stacl} is the most common latency metric; it sometimes results in negative values, as noted by \citeA{kano2024atd}.} 
Note that to generate outputs from the proposed model, we applied \texttt{<off>} tags for the offline test and \texttt{<si>} tags for the SI reference and CMT reference tests. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Results}
\label{sec:results}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Full-sentence Offline Translation Results}

Table~\ref{tab:generate_offline_main_result} lists the experimental results of full-sentence offline translation in BLEURT, COMET-QE and BLEU for the Offline, SI reference and CMT reference test sets. 
Note that COMET-QE in the SI and COMET-QE in CMT reference tests had the same results because the same source transcripts were used in both the SI reference and CMT reference tests. 
Additionally, the number of source transcripts in the offline test differed from those in the SI reference and CMT reference tests, which resulted in different COMET-QE scores.
These results show that the proposed Style FT and $\text{Style-MultiAug-4}$ FT surpassed the baselines in BLEURT and COMET-QE for all test sets.
This finding suggests that our proposed Style FT and $\text{Style-MultiAug-4}$ FT can produce more semantically aligned outputs than the SI FT and Offline FT baselines in the SI reference and CMT reference tests. 
By contrast, the Offline FT and SI FT baselines surpassed the proposed Style FT and $\text{Style-MultiAug-4}$ FT in BLEU.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 4
\begin{table}[t]
\input{01table04.tex}%
\hangcaption{BLEURT, COMET-QE and BLEU in full-sentence offline ST on Offline, SI reference and CMT reference test sets.}
\label{tab:generate_offline_main_result}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 4
\begin{figure}[t]
\begin{center}
\includegraphics{32-2ia1f4.eps}
\end{center}
%%%%\subcaption{BLEURT in Offline test}
%%%%\label{fig:sharedtask-simulst-LA-tst-common-xATD-yBLEURT}: a
%%%%\subcaption{COMET-QE in Offline test}
%%%%\label{fig:sharedtask-simulst-LA-tst-common-xATD-yCOMETQE}: b
%%%%\subcaption{BLEU in Offline test}
%%%%\label{fig:sharedtask-simulst-LA-tst-common-xATD-yBLEU}: c
%%%%\subcaption{BLEURT in SI reference test}
%%%%\label{fig:sharedtask-simulst-LA-test-xATD-yBLEURT}: d
%%%%\subcaption{COMET-QE in SI reference test}
%%%%\label{fig:sharedtask-simulst-LA-test-xATD-yCOMETQE}: e
%%%%\subcaption{BLEU in SI reference test}
%%%%\label{fig:sharedtask-simulst-LA-test-xATD-yBLEU}: f
%%%%\subcaption{BLEURT in CMT reference test}
%%%%\label{fig:sharedtask-simulst-LA-test-xATD-yBLEURT-CMT}: g
%%%%\subcaption{COMET-QE in CMT reference test}
%%%%\label{fig:sharedtask-simulst-LA-test-xATD-yCOMETQE-CMT}: h
%%%%\subcaption{BLEU in CMT reference test}
\caption{SimulST latency (ATD) -- quality results on test sets.}
\label{fig:main-result}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Simultaneous Translation Results}

Figure~\ref{fig:main-result} shows the quality and latency plots for SimulST in BLEURT, COMET-QE, and BLEU on the Offline, SI reference, and CMT reference test sets.
Each point in each graph represents the score for speech segment sizes of 200, 400, 600, 800, and 1,000~ms from the left.
%%%%In Figures~\ref{fig:sharedtask-simulst-LA-tst-common-xATD-yBLEURT}, \ref{fig:sharedtask-simulst-LA-test-xATD-yBLEURT} and \ref{fig:sharedtask-simulst-LA-test-xATD-yBLEURT-CMT}, 
In Figures~\ref{fig:main-result}a, \ref{fig:main-result}d and \ref{fig:main-result}g, 
the proposed Style FT and Style-MultiAug-4 FT clearly exhibit better BLEURT results than the Offline FT, SI FT, and Mixed FT baselines. 
%%%%Note that the COMET-QE in the SI reference test (Figure \ref{fig:sharedtask-simulst-LA-test-xATD-yCOMETQE}) 
    Note that the COMET-QE in the SI reference test (Figure \ref{fig:main-result}e) 
%%%%and the COMET-QE in CMT reference test (Figure \ref{fig:sharedtask-simulst-LA-test-xATD-yCOMETQE-CMT}) had the same results 
    and the COMET-QE in CMT reference test (Figure \ref{fig:main-result}h) had the same results 
because the same source transcripts were used for both the SI reference and CMT reference tests.
According to the COMET-QE in the Offline test (Figure \ref{fig:main-result}b), the Offline FT baseline was better than the proposed Style FT and Style-MultiAug-4 FT. 
%%%%From the COMET-QE in the SI reference test (Figure \ref{fig:sharedtask-simulst-LA-test-xATD-yCOMETQE}), 
    From the COMET-QE in the SI reference test (Figure \ref{fig:main-result}e), 
compared with the Offline FT baseline, the proposed Style FT exhibited low latency, and the Style-MultiAug-4 FT performed better in all latency ranges. 
It can be seen from the COMET-QE in the Offline and SI reference tests 
%%%%(Figures \ref{fig:sharedtask-simulst-LA-test-xATD-yCOMETQE} and \ref{fig:sharedtask-simulst-LA-tst-common-xATD-yCOMETQE}) 
    (Figures \ref{fig:main-result}e and \ref{fig:main-result}b) 
that both SI FT baselines were worse than the other models. 
These results suggest that a direct SI FT causes inappropriate omissions and fails to retain speaker content. 
In contrast, the Offline FT could learn to retain the speaker's content, resulting in the best COMET-QE scores in the Offline test 
%%%%(Figure \ref{fig:sharedtask-simulst-LA-tst-common-xATD-yCOMETQE}). 
    (Figure \ref{fig:main-result}b). 
However, our proposed Style FT and Style-MultiAug-4 FT could also avoid excessive translation omissions compared to the SI FT baseline, even when using SI data, and they could retain the speaker's content as effectively as that in Offline FT. 
In particular, Style-MultiAug-4 FT demonstrates a higher tendency to achieve this than Style FT.
From the overall results of BLEURT and COMET-QE, Style-MultiAug-4 FT performed better than Style FT. 
%%%%However, when we observed the BLEU trends in the SI reference test (Figure \ref{fig:sharedtask-simulst-LA-test-xATD-yBLEU}), 
    However, when we observed the BLEU trends in the SI reference test (Figure \ref{fig:main-result}f), 
the baseline SI FT was the best in all latency ranges. 
%%%%The CMT reference test results (Figures \ref{fig:sharedtask-simulst-LA-test-xATD-yBLEURT-CMT}, \ref{fig:sharedtask-simulst-LA-test-xATD-yCOMETQE-CMT} and \ref{fig:sharedtask-simulst-LA-test-xATD-yBLEU-CMT}), 
    The CMT reference test results (Figures \ref{fig:main-result}g, \ref{fig:main-result}h and \ref{fig:main-result}i), 
BLEURT and BLEU trends were approximately similar. 
These results showed similar trends; Style-MultiAug-4 FT and Style FT were better than the other baselines, and Style-MultiAug-4 FT was better than Style FT.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 5
\begin{table}[b]
\input{01table05.tex}%
\hangcaption{Fluency and adequacy evaluated by human simultaneous interpreter. Each set of sentences consists of all 511 sentences generated from each ST model for the SI reference test evaluation. All three results are from when the speech segment size is 400ms.
The values mean averaged score and () means variance. }   
\label{tab:human_evaluation}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Human Evaluation by Simultaneous Interpreter}

We also conducted a human evaluation by a native Japanese simultaneous professional interpreter to determine which model's output was more SI-like in the following two points. 
\begin{description}
    \item[Adequacy] To check whether the translation is making inappropriate omissions. To identify the source, the speech content was retained in the translation.  
    \item[Fluency] To check if the translation is grammatically fluent, such as in full-sentence offline translation.  
\end{description}
The interpreter presented three output sentences generated by the (1) baseline SI FT, (2) proposed Style FT, and (3) Style-MultiAug-4 FT models simultaneously without providing information on how the translation was generated. We asked the interpreter to assign a score of to 1-5 to the three sentences for each evaluation point: adequacy and fluency. 
Each set of sentences comprised 511 sentences generated by each ST model for the SI reference test evaluation. 
Table~\ref{tab:human_evaluation} shows the adequacy and fluency results of the baseline SI FT, proposed Style FT, and $\text{Style-MultiAug-4}$ FT methods evaluated by a simultaneous interpreter. 
These three results were obtained for a speech segment size is 400~ms.
The results show that our proposed methods obtained better scores in terms of adequacy and fluency, and $\text{Style-MultiAug-4}$ FT obtained better scores than Style FT. 
The gap between $\text{Style-MultiAug-4}$ FT and Style FT was larger in adequacy (4.23 vs. 4.08) than in fluency (4.29  vs. 4.24). 
We also observed that the gap between the proposed $\text{Style-MultiAug-4}$ and the baseline SI FT was more significant in adequacy (4.23 vs. 3.57) compared to that in fluency (4.29 vs. 3.83). 
In this study, we focused on alleviating undergeneration in the baseline model, and it showed that adequacy improved significantly compared with fluency, because we did not focus directly on fluency in this study. 
However, we observed improvements in both fluency and adequacy. 
This indicates the effectiveness of our proposed methods not only for maintaining the source content, but also for maintaining the naturalness of the translation. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Discussions}
\label{sec:discussions}

The main results in Section \ref{sec:results} demonstrate the overall effectiveness of the proposed Style FT and Style-MultiAug-4 FT. 
However, it is not clear whether applying self-training in multiple stages is truly effective, as it remains unclear whether the selected $N=4$ model outperforms the intermediate models in $N=1, 2, 3$ or the models trained with upsampling approach using the original SI data.
Intermediate models, such as Style-MultiAug-2 or Style-MultiAug-3, which were trained in the process of training Style-MultiAug-4, may also be sufficient. 
A simple upsampling approach using the original SI data may be sufficient for using pseudo-SI data in multistage self-training. 
By contrast, although the proposed methods were good overall, there were cases where the trend differed depending on the test set and evaluation metrics. 
For example, in BLEU, the baseline SI FT performed better on the SI reference test. 
We attribute these trend differences to variations in the outputs of the baseline and proposed methods as well as differences in the test sets.
To clarify these points in detail, we conducted the following analyses: 

\begin{enumerate}
    \item We examined the score trends in intermediate models ($N=1, 2, 3$) in multistage self-training to examine how the scores changed in each stage.
    \item We observed the results when applying upsampling to examine the effectiveness of using pseudo-SI data.
    \item We compared the output trends in the baseline and proposed methods and test sets differences to examine where the result trend differences came from. 
\end{enumerate}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 5
\begin{figure}[t]
\begin{center}
\includegraphics{32-2ia1f5.eps}
\end{center}
%%%%\subcaption{BLEURT in Offline test}
%%%%\label{fig:sharedtask-simulst-LA-test-xATD-yBLEURT-offline-multieffect}: a
%%%%\subcaption{BLEURT in SI reference test}
%%%%\label{fig:sharedtask-simulst-LA-test-xATD-yBLEURT-SI-multieffect}: b
%%%%\subcaption{BLEURT in CMT reference test}
%%%%\label{fig:sharedtask-simulst-LA-test-xATD-yBLEURT-CMT-multieffect}: c
\hangcaption{SimulST latency (ATD) -- BLEURT result in Offline, SI reference and CMT reference tests to examine the effectiveness of multistage self-training. }
\label{fig:sharedtask-simulst-LA-test-xATD-yBLEURT-multieffect}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Effectiveness of Multistage Self-training}

To determine the effectiveness of multistage self-training, we compared the translation performance of the Style-MultiAug-$N$ FT models with different $N$, specifically, $N=1$ to $N=5$.
Figure~\ref{fig:sharedtask-simulst-LA-test-xATD-yBLEURT-multieffect} shows the BLEURT results of multistage self-training from $\text{Style-MultiAug-1}$ FT to $\text{Style-MultiAug-5}$ FT in the Offline, SI reference and CMT reference test sets. 
Among all test sets, the performance improved progressively when $N$ increased. 
These results show that as the number of stages increases, the model performance improves. 
We expected the model to be trained with different variations in the pseudo-SI data in each step. 
%%%%In the Offline test (Figure~\ref{fig:sharedtask-simulst-LA-test-xATD-yBLEURT-offline-multieffect}), 
    In the Offline test (Figure~\ref{fig:sharedtask-simulst-LA-test-xATD-yBLEURT-multieffect}a), 
although there was no significant performance improvement compared with the CMT reference and SI reference tests, there was some improvement in performance. 
The slight improvement in the offline test can be attributed to the increasing amount of SI data as the stages progressed, whereas the amount of offline data remained unchanged.
Nevertheless, we found that multistage self-training was effective in the Offline test. 
In this study, we selected Style-MultiAug-4 FT as the best model based on the development loss across stages from $N=1$ to $N=5$. 
Although the results from $N=5$ showed a slight improvement over $N=4$ in the CMT reference and Offline tests, the SI reference test results remained largely unchanged between $N=4$ and $N=5$. 
Based on the evaluation results, the selection of models based on development loss during the multistage self-training process appears to be appropriate.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{ Effectiveness of Upsampling}

Upsampling is one of the easiest approaches for increasing the contribution of small data when training with both large and small data. 
We can also upsample the SI data instead of using augmented data with self-training in $\text{Style-MultiAug-4}$ FT. 
We also conducted experiments under the following settings to determine the effect of SI data upsampling. 

\begin{description}
    \item[Mixed-Up FT] Fine-tuned using both the offline and SI data. The SI portions were upsampled when fine-tuned from both the offline and SI data without style tags.
    \item[Style-Up FT] The SI portions were upsampled when fine-tuning using both the offline and SI data with style tags.
\end{description}

From the results in Table~\ref{tab:generate_offline_main_result_upsampling}, we compared the Mixed FT with Mixed-Up FT or Style FT with Style-Up FT and observed that simply upsapling SI did not improve the model performance. 
The Mixed-Up FT was better than the Mixed FT in the SI reference test BLEU, but was worse in the Offline test BLEU as a trade-off. 
We also observed the same trend when comparing the Style-Up FT with the Style FT.  
In addition, we observed that the Style-Up FT with simple upsampling did not surpass the Style-MultiAug-4 FT. 
These results show that the upsampling of small amounts of SI data was ineffective for both the previous and proposed methods, which could be due to slight variations in the upsampled SI data.
In contrast, the Style-MultiAug-4 FT uses diverse augmented pseudo-SI data. 
This would have resulted in an improvement over the Style FT and Style-Up FT.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 6
\begin{table}[t]
\input{01table06.tex}%
\hangcaption{BLEURT, COMET-QE and BLEU results in the upsampling methods in full-sentence offline ST on Offline, SI reference, and CMT reference test sets. }
\label{tab:generate_offline_main_result_upsampling}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Output trends from baseline and proposed models}

To clarify the performance differences in the test sets and evaluation metrics, we first observed the output trends in the baseline and proposed models through the length differences. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Length differences}

First, we focus on the length differences between the translation outputs and references.
Figure~\ref{fig:sharedtask-simulst-xATD-yLengthRatio-CMT} shows the length ratios between the translation results and CMT reference test. 
As shown in Figure~\ref{fig:sharedtask-simulst-xATD-yLengthRatio-CMT}, the proposed Style FT and Style-MultiAug-4 FT resulted in longer outputs than those of the baselines, and the SI FT baseline preferred a shorter output than the others and references. 
In addition, the length ratio in the SI reference test was higher than that in the CMT reference test. 
This is probably because SI sentences generated by human interpreters tend to contain many omissions. 
Moreover, because the SI FT is trained with SI data directly, it tends to be overfitted and produces significantly short outputs. 
In contrast, our proposed Style FT and Style-MultiAug-4 FT prefer longer outputs, alleviating the overfitting problem associated with short outputs. 
This suggests that the baseline SI FT suffers from undertranslation, and the proposed Style FT and Style-MultiAug-4 FT methods alleviate such undertranslation problems. 
However, the proposed Style FT and Style-MultiAug-4 FT methods exhibited overgeneration trends, especially in the low-latency range.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 6
\begin{figure}[t]
\begin{center}
\includegraphics{32-2ia1f6.eps}
\end{center}
\caption{Length ratio results in CMT reference test.}
\label{fig:sharedtask-simulst-xATD-yLengthRatio-CMT}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Undergeneration in SI FT baseline results in inflated BLEU scores in the SI reference test}

We observed undergeneration trends in the baseline SI FT methods and overgeneration trends in the proposed Style FT and Style-MultiAug-4 FT methods. 
We believe that these output trends can lead to different results for different test sets and evaluation metrics. 
When we compared the trend in BLEU of the SI reference test and BLEU of the CMT reference test, the large difference was in the reference length between the SI reference and CMT reference tests.
There were almost no omissions in the CMT reference test; however, there were omissions in the SI reference test. 
Then, in the case of short outputs from a SimulST model, even if the necessary content is included, there can be a smaller penalty for short output in BLEU. 
Therefore, it is difficult to evaluate SimulST for SI tasks using BLEU considering omissions. Therefore, a method to evaluate the similarity between sentences and words may be more suitable for interpretation.
First, to confirm the trends in both output length and word similarity, we evaluated the BERTScore \cite{DBLP:journals/corr/abs-1904-09675-bertscore} results for F1, recall, and precision.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%% fig. 7
\begin{figure}[b]
\begin{center}
\includegraphics{32-2ia1f7.eps}
\end{center}
%%%%\subcaption{BERTScore F1 in SI reference test}
%%%%\label{fig:BERTScoreF1-SI}: a
%%%%\subcaption{BERTScore Recall in SI reference test}
%%%%\label{fig:BERTScoreF1Recall-SI}: b
%%%%\subcaption{BERTScore Precision in SI reference test}
%%%%\label{fig:BERTScorePrecision-SI}: c
%%%%\subcaption{BERTScore F1 in CMT reference test}
%%%%\label{fig:BERTScoreF1-CMT}: d
%%%%\subcaption{BERTScore Recall in CMT reference test}
%%%%\label{fig:BERTScoreF1Recall-CMT}: e
%%%%\subcaption{BERTScore Precision in CMT reference test}
%%%%\label{fig:BERTScorePrecision-CMT}: f
\caption{SimulST latency (ATD) -- quality (BERTScore) results on test sets.}
\label{fig:bertscore}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Figure~\ref{fig:bertscore} shows the detailed results for F1, recall, and precision using BERTScore for the SI reference and CMT reference test sets. 
This indicates that the F1 scores in the proposed models were better in both the SI reference and CMT reference tests, particularly in the low-latency range.
When we observed the recall, the proposed Style-MultiAug-4 FT performed the best in BERTScore recall in the SI. 
We expect that longer outputs in the proposed Style-MultiAug-4 FT will lead to a recall gain. 
By contrast, the SI FT baseline was the best for BERTScore precision in the SI reference test. 
%%%%The precision trend in the SI reference test (Figure \ref{fig:BERTScorePrecision-SI}) was very similar to that of BLEU in the SI reference test 
    The precision trend in the SI reference test (Figure \ref{fig:bertscore}c) was very similar to that of BLEU in the SI reference test 
%%%%(Figure \ref{fig:sharedtask-simulst-LA-test-xATD-yBLEU}).
    (Figure \ref{fig:main-result}f).
These results suggest that BERTScore precision and BLEU scores are influenced by the output length. 
From the perspective of the precision of the translation results, outputs that were longer than their references were unfavorable.
In addition, the SI reference test set contained several omissions. 
Usually, BLEU penalizes a short output via a brevity penalty, but the penalty for short hypotheses was not sufficiently applied because the references were originally short owing to omissions. 
Consequently, the SI FT results were overestimated by BLEU in the SI reference test.

In the CMT reference test, all results showed the same trend as in BLEURT, COMET-QE, and BLEU, indicating that the proposed Style FT and Style-MultiAug-4 FT methods were better. 
This also indicates that the SI reference test is not suited to BLEU evaluation because it is easily affected by the under-generation effect of the baseline SI FT. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Non-speech events causes over-generation in proposed methods in low latency range}

We observed overgeneration trends in the proposed Style FT and Style-MultiAug-4 FT methods. 
As shown in Figure~\ref{fig:sharedtask-simulst-xATD-yLengthRatio-CMT}, we observed overtranslation in the proposed methods, especially in the case of low latency, that is, when the speech segment size was 200~ms. 
We observed some repetitions by the proposed method, such as (拍手) (拍手) ..., which means (Applause). 
These nonspeech sound events (applause and laughter) are often found in TED Talks and are transcribed as labels in the target translation of the offline data.
When a model is directly fine-tuned on SI data that contain no nonspeech event labels, the outputs of the model include few such labels.
By contrast, our proposed models generated a significant number of nonspeech sound labels and repetitions related to the labels.
We attribute this difference to the fact that the proposed models are trained with both mixed offline and SI data simultaneously, making them more influenced by nonspeech event labels in the offline data, even when focusing on the SI-style output with style tags.
Based on this assumption, we attempted to eliminate typical repetitions as follows and then conducted an evaluation: 

\begin{itemize}
    \item Remove tokens if they are surrounded by ``()'' and ``\texttt{<>}''. (if the tokens include parts of ``(拍手)'' like ``拍手)'' or ``('', they were also excluded.)
    \item Stop generating output as soon as any 3-gram appears at least three times before reaching the end of the sentence.
\end{itemize}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 8
\begin{figure}[t]
\begin{center}
\includegraphics{32-2ia1f8.eps}
\end{center}
%%%%\subcaption{BLEU in CMT reference test}
%%%%\label{fig:sharedtask-simulst-LA-test-rmrep-bleu}: a
%%%%\subcaption{Length ratio in CMT reference test}
%%%%\label{fig:sharedtask-simulst-LA-test-rmrep-lengthratio}: b
%%%%\subcaption{BLEURT in CMT reference test}
%%%%\label{fig:sharedtask-simulst-LA-test-rmrep-bleurt}: c
%%%%\subcaption{BERTScore F1 in CMT reference test }
%%%%\label{fig:sharedtask-simulst-LA-test-rmrep-bertscore-f1}: d
%%%%\subcaption{BERTScore Precision in CMT reference test}
%%%%\label{fig:sharedtask-simulst-LA-test-rmrep-bertscore-precision}: e
%%%%\subcaption{BERTScore Recall in CMT reference test}
%%%%\label{fig:sharedtask-simulst-LA-test-rmrep-bertscore-recall}: f
\hangcaption{Results with repetition removal (Rmrep) in BLEU, length ratio, BLEURT and BERTScore scores in ATD on CMT reference test set. }
\label{fig:sharedtask-simulst-LA-test-rmrep-bleu-len-ratio-bleurt-bertscore}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We applied this repetition removal to the results of the SI FT, Style FT, and $\text{Style-MultiAug-4}$ FT, which are labeled SI-Rmrep FT, Style-Rmrep FT, and $\text{Style-MultiAug-4}$-Rmrep FT, respectively. 
Figure~\ref{fig:sharedtask-simulst-LA-test-rmrep-bleu-len-ratio-bleurt-bertscore} shows the BLEU, length ratio, BLEURT, and BERTscore scores before and after the repetition removal. 
We observed a larger length reduction when the speech segment size was 200~ms in the proposed Style FT and $\text{Style-MultiAug-4}$ FT compared with the baseline SI FT. 
The repetition effect was lower for the baseline SI FT than that for the proposed model. 
In both the proposed Style FT and $\text{Style-MultiAug-4}$ FT, BLEUs increased, whereas the SI FT baseline did not improve when the speech segment size was 200~ms. 
This suggests the existence of several repetitions in the translation results of the proposed method. 
In BLEURT, the baseline was lower than those of the other proposed models when the speech segment size was 200~ms. 
The BERTScore F1 was lower than SI FT overall compared to that for the SI-Rmrep FT; the scores between Style FT and Style-Rmrep FT or between $\text{Style-MultiAug-4}$ FT and $\text{Style-MultiAug-4}$-Rmrep FT were approximately similar, respectively. 
This could be due to the greater score reduction of SI-Rmrep FT and the same scores between Style FT and Style-Rmrep FT or between $\text{Style-MultiAug-4}$ FT and $\text{Style-MultiAug-4}$-Rmrep FT in BERTScore precision. 
Overall, these findings suggest an overtranslation trend from the repetitions caused by nonspeech event labels in the proposed methods in the low-latency range. 
Nevertheless, we observed no significant impact on semantic similarity, as it does not change significantly before and after excluding these repetitions.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Output examples}

Table~\ref{tab:example-sentence-baseline-proposal-400ms} shows example sentences from the baseline SI FT, proposed Style FT, and Style MultiAug-4 FT with 400-ms segment size. 
Each human evaluation score is also included. 
In Example 1, although the SI FT is undertranslated, the SI reference test translation is also excessively short.
The SI FT was also trained directly on SI training data containing such excessively short SI translations, and we assume that the SI FT model was omitted excessively without understanding which parts should be omitted. 
However, from the source and the CMT reference test, the proposed Style FT and Style-MultiAug-4 FT retained more speaker content.
The human evaluation scores of the proposed methods were higher than those of the baseline SI FT. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 7
\begin{table}[t]
\input{01table07.tex}%
\hangcaption{Example sentences in SI FT, Style FT and $\text{Style-MultiAug-4}$ FT (speech segment size: 400ms) on SI reference and CMT reference test sets. Ade. means Adequacy and Flu. means Fluency. () is the English translation result translated by DeepL in 2024/08/06. }
\label{tab:example-sentence-baseline-proposal-400ms}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Example 2 also shows that the part ``they were hacked into'' was not translated in the baseline SI FT, indicating the under-translation problem. 
However, we see that the outputs of the proposed method are longer than those of the baseline SI FT, and this trend is more visible in the $\text{Style-MultiAug-4}$ FT than in the Style FT. 
However, the output tends to include more connection words and subjects, such as ``これは (this),'' ``なぜなら (because),'' and ``彼らは (they).'' 
This did not significantly affect the communication of meaning. 
Even if these words are removed, their meanings are still understood in many cases and do not significantly affect the performance. 
This suggests that the over-generated lengthy parts of the proposed method have no significant impact on the meaning of the translation. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}

In this study, we proposed an effective method for training a SimulST model using mixed data of SI- and offline-style translations with style tags to train the model to generate outputs in either style, motivated by the tag-based approach in domain adaptation.
Experimental results on the English-to-Japanese SimulST demonstrated the overall advantage of the proposed method, particularly in BLEURT and COMET-QE in the CMT reference test set.  
Future work will include training a SimulST model that makes appropriate omissions, such as SI generated by human interpreters. 
Our proposed models retained more content in the source speech but tended to produce a longer output than the baseline models. 
For continuous and longer speech, a SimulST model should also be evaluated to determine whether it can keep up with the source speech.
In future studies, we will also consider applying the proposed method to other language pairs. 
We used the NAIST-SIC-Aligned \cite{zhao-etal-2024-naist} and English-Japanese SI data aligned at the utterance level to investigate the effectiveness of our proposed methods. 
These methods can also be applied if the utterance-level SI data are available for other language pairs. 
In addition, in this study, pseudo-SI data in self-training were generated from offline ST data's speech, but future work will also explore the generation of more from monolingual speech data. 
Recent advances in large language models (LLMs) have shown promise in translation tasks. 
We consider leveraging LLMs to extract distinctive characteristics from limited SI data and develop a SimulST system that properly handles appropriate omissions.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment

Part of this work was supported by JSPS KAKENHI Grant Numbers JP21H05054, \linebreak JP23K21681 and JP24KJ1695.
This paper is an extended version of that published in Proceedings of the 20th International Conference on Spoken Language Translation \cite{ko-etal-2023-tagged}. 
In this study, we expand on the ideas of previous works and conduct an additional experiment to reveal the effectiveness of our proposed multistage self-training method. 
We also conducted further analyses in COMET-QE on CMT reference test set to determine the effectiveness of our proposed method in maintaining source content in translations.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.7}
\bibliography{01refs}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Yuka Ko}{
She is now a Ph.D. student in the Human-AI Interaction Laboratory, Nara Institute of Science and Technology. 
She received her bachelor's degree in engineering from the Osaka Prefecture University in 2020 and her master's degree in 2022 from the Nara Institute of Science and Technology. 
She is a recipient of the JSPS Research Fellowship for Young Scientist DC2. 
Her research interests include speech translation, simultaneous translation, and spoken language processing. 
}

\bioauthor[:]{Ryo Fukuda}{
He is a researcher at the Signal Processing Research Group, Media Information Laboratory, NTT Communication Science Laboratories. 
He received his bachelor's degree in engineering from the Toyohashi University of Technology in 2019 and his master's degree and doctor's degree in engineering from the Nara Institute of Science and Technology in 2021 and 2023. 
}

\bioauthor[:]{Yuta Nishikawa}{
He received his bachelor's degree in engineering from the National Institute of Technology, Nara College in 2022 and his master's degree in engineering from the Nara Institute of Science and Technology in 2024.
Since 2024, he has been the CEO of CueBeX Inc.
}

\bioauthor[:]{Yasumasa Kano}{
He received his bachelor's degree in economics from \linebreak\mbox{Yokohama} National University in 2019, and his master's and Ph.D. degrees in engineering in 2022 and 2024, respectively, from Nara Institute of Science and Technology.
He was the CTO of Wonder Palette Inc. in 2024.
Since 2021, he has been the CEO of Tleez Inc.
He is a member of ANLP.
}


\bioauthor[:]{Katsuhito Sudoh}{%
He is a professor at Nara Women's University.
He received his bachelor's degree in engineering in 2000, and his master's and Ph.D. degrees in informatics in 2002 and 2015, respectively, from Kyoto University.
He worked with the NTT Communication Science Laboratories from 2002 to 2017 and the Nara Institute of Science and Technology from 2017 to 2024.
He works on machine translation and natural language processing.
He is a member of ACL, ISCA, ANLP, IPSJ, ASJ, and JSAI.
}

\bioauthor[:]{Sakriani Sakti}{
She is the Head of the Human-AI Interaction (HAI) Research Laboratory and a Full Professor at NAIST, Japan. She also holds adjunct positions at JAIST and the University of Indonesia and is a Visiting Research Scientist at RIKEN AIP.
She earned her B.E. from the Bandung Institute of Technology, Indonesia (1999), and her M.Sc. and Ph.D. from the University of Ulm, Germany (2002, 2008). From 2003 to 2011, she was a researcher at ATR and NICT, Japan, before joining NAIST, where she advanced from Assistant Professor to her current role. She has been actively involved in international collaborations, including the Asian Pacific Telecommunity Project, A-STAR, and U-STAR. She has also served as an IEEE SLTC Committee Member and an Associate Editor for IEEE/ACM TASLP. Currently, she chairs ELRA/ISCA SIGUL and serves as the Convener of O-COCOSDA.
Her research focuses on multilingual and multimodal spoken language processing, deep learning, and cognitive communication.
}


\bioauthor[:]{Satoshi Nakamura}{%
He is a Professor Emeritus at the Nara Institute of Science and Technology NAIST, a full professor at the Chinese University of Hong Kong, Shenzhen CUHKSZ, and an honorary professor at the Karlsruhe Institute of Technology, Germany.
He received his B.S. degree from the Kyoto Institute of Technology in 1981 and a Ph.D. from the Kyoto University in 1992.
He was the Director of the ATR SLC laboratory from 2000 to 2008 and the Director General of Keihanna Research Laboratories, NICT.
He was a professor and Director of the AHC Laboratory at NAIST from 2011 to 2023.
He is currently a full professor at CUHKSZ.
He was an elected board member of the ISCA in the period of June 2011–2019, an IEEE Signal Processing Magazine Editorial Board member in the period 2012--2015, and an IEEE SPS Speech and Language Technical Committee Member in the period 2013--2015.
He is an ATR Fellow, IPSJ Fellow, IEEE Fellow, and ISCA Fellow.
}
\end{biography}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\biodate



\clearpage























\clearpage



















\end{document}


