\documentclass[english]{jnlp_3.8}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{array}
\allowdisplaybreaks

\usepackage{hangcaption_jnlp}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition

\usepackage{makecell}
%%%%%\usepackage{xcolor}
%%%%%\allowdisplaybreaks

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{31}
\Number{3}
\Month{September}
\Year{2024}
%%%%\Issuetitle{「　　　　」特集号}

\received{2023}{11}{15}
\revised{2024}{3}{1}
\accepted{2024}{4}{8}


\setcounter{page}{935}



%%%%   Title, Author, etc.            %%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{End-to-end Japanese-English Speech-to-text Translation \\ with Spoken-to-Written Style Conversion}
\eauthor{Zhengdong Yang\affiref{Author_1} \and Shuichiro Shimizu\affiref{Author_1} \and Chenhui Chu\affiref{Author_1} \and Sheng Li\affiref{Author_2} \and Sadao Kurohashi\affiref{Author_1}} 
\eabstract{%
Speech-to-text translation (ST) translates speech from the source language into text in the target language. Because ST deals with different forms of language, it faces a language style gap between spoken and written language. The gap lies not only between the input speech and the output text but also between the input speech and the bilingual parallel corpora that are often used in ST. These gaps become an obstacle to improving the performance of ST. Spoken-to-written style conversion has been proven to improve cascaded Japanese-English ST by reducing such gaps. Integrating this conversion into end-to-end ST is desirable because of its ease of deployment, improved efficiency, and reduced error propagation compared to cascaded ST. In this study, we construct a large-scale Japanese-English lecture domain ST dataset. We also propose a joint task of speech-to-text spoken-to-written style conversion and end-to-end ST, as well as an interactive-attention-based multi-decoder model for the joint task to improve end-to-end ST. Experiments on the constructed dataset show that our model outperforms a strong baseline.
}
\ekeywords{Speech Translation, Spoken-To-Written Style Conversion}

\headauthor{Yang et al.}
\headtitle{E2E ST with Spoken-to-Written Style Conversion}



%%%%     Affiliation     %%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\affilabel{Author_1}{}{Graduate School of Informatics, Kyoto University}
\affilabel{Author_2}{}{National Institute of Information and Communications Technology}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S1
\section{Introduction}
Speech-to-text translation (ST) translates source language speech into target language text \cite{Stentiford1988}.
As the input and output forms are different, in many cases, their language styles are also different.
Input speech is spoken and likely to contain many colloquial expressions, omissions, and verbose sentences, which rarely appear in the written output text.
Except in cases where retaining informality in the content is desirable, the features and expressions in the spoken style are redundant or even detrimental to delivering accurate semantic information in the translated text.

In addition to the gap between the language styles of ST input and output, the gap between the language styles of ST input and the bilingual parallel corpora must also be noted. Although bilingual parallel corpora are mainly used for text-to-text machine translation (MT), some of their forms are occasionally used for ST \cite{Stentiford1988,alinejad-sarkar-2020-effectively,DBLP:conf/interspeech/LiuXZHWWZ19} 
because MT and ST are highly related. 
However, the language-style gap is an obstacle to exploiting bilingual parallel corpora to improve ST performance.

A solution to the aforementioned problems involving language styles is to combine ST with spoken-to-written style conversion. This approach involves converting spoken-language text into written-language text while maintaining identical semantic meaning. 
It has been shown that spoken-to-written style conversion (hereinafter referred to as \emph{S2WConv}) improves the accuracy of Japanese-English cascaded speech translation \cite{nakao2021}, which divides the task into 2 steps: automatic speech recognition (ASR) and MT. S2WConv acted as an intermediate step between ASR and MT. It reduces the gap between the spoken language in the transcriptions generated by ASR systems and the written language in the datasets used for training MT systems. See Table \ref{table0} for an example.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T1
\begin{table}[b]
\input{04table01.tex}
\hangcaption{An example of S2WConv (differences are underlined). Compared to the original spoken-style Japanese transcription in spoken language, the written-style transcription is closer to the language style in bilingual corpora for MT. Thus, it is more likely to be machine-translated correctly.}
\label{table0}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

However, because of the limitations of the cascaded ST, integrating S2WConv into the 
end-to-end ST is desirable.
End-to-end ST \cite{Berard2016ListenAT} directly generates translations from speech without 
intermediate steps. Compared with cascade ST, it has the advantages of
easier deployment, higher efficiency, and less error propagation.
Owing to the lack of large-scale ST corpora, existing end-to-end ST methods require pre-training or multi-task learning with ASR and MT. Therefore, using S2WConv to improve end-to-end ST is reasonable because the language style gap between speech transcription and MT training data is present in end-to-end ST.

In this study, we propose a multi-task learning model that integrates S2WConv into the end-to-end ST.
Unlike previous ST multi-task learning methods, we utilize speech-to-text spoken-to-written style conversion (hereinafter referred to as SS2WConv), where the input is speech and the output is the written-style transcript in the source language.
The model has a dual-decoder architecture with an interactive attention mechanism that allows interactions between different decoders.
This reduces the aforementioned language-style gaps under the premise of adopting MT pre-training with bilingual parallel corpora.
To obtain data on written-language transcriptions, we propose synthesizing them using a pre-trained S2WConv model trained on a smaller textual conversion dataset.
We also constructed a large-scale lecture domain ST dataset on Japanese-English language pair to experimentally verify our model's effectiveness.
Our results demonstrate the effectiveness of the proposed model when adopting MT pre-training with written-style data.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2
\section{Related Work}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2.1
\subsection{Spoken-to-Written Style Conversion (S2WConv)}
Several methods have been proposed for S2WConv in previous works, including weighted finite-state transducers \cite{6289034}, pointer-generator networks \cite{9053930} and Transformer-based models \cite{nakao2021}. 
We adopted the last method and integrated it with the end-to-end ST.

There have also been studies in which efforts have been made to build corpora dedicated to this task. For Japanese, Ihori et al. \citeyear{ihori-etal-2020-parallel} built a corpus by converting spoken-style text with a crowdsourcing service. 
A similar method was used by Nakao et al. \citeyear{nakao2021} to construct the corpus adopted in this study. 
For other languages, Dominutti et al. \citeyear{DBLP:conf/clic-it/DominuttiPDMQ19} conducted a pilot study to build an Italian written-spoken parallel corpus by aligning broadcast transcriptions with news articles. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2.2
\subsection{Multi-task Learning of ASR and ST}
Because the performance of simple end-to-end ST is generally limited, several studies have proposed the use of a multi-task learning approach \cite{DBLP:journals/ml/Caruana97} to improve the translation quality of end-to-end ST. Weiss et al. \citeyear{DBLP:conf/interspeech/WeissCJWC17} first investigated this subject and found that a multi-task ASR and ST model performs better than a multi-task MT and ST model. In their method, only the encoder is shared, and the decoders for different tasks cannot utilize each other's information. A two-stage model has been proposed to address this problem \cite{DBLP:conf/interspeech/KanoS017,DBLP:conf/interspeech/Anastasopoulos018,DBLP:journals/tacl/SperberNNW19}.
It performs ASR and then passes the decoder states as inputs to the ST decoder. However, it has limited efficiency in the training and inference processes. 

To address the limitations of the aforementioned methods, Liu et al. \citeyear{DBLP:conf/aaai/LiuZXZHWWZ20} proposed an interactive attention mechanism.
This mechanism enables synchronous ASR and ST with information exchange. The ASR and ST decoders interact during decoding. Both decoders rely on their respective outputs, as well as the outputs produced by the other decoders.

Inspired by Liu et al. \citeyear{DBLP:conf/aaai/LiuZXZHWWZ20}, Le et al. \citeyear{DBLP:conf/coling/LePWGSB20} proposed the dual-decoder Transformer. It has a more general framework with different variants and combinations of settings for interactive attention mechanisms. We were inspired by their work and further improved the performance of their model by integrating S2WConv.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3
\section{Dataset Construction}

To evaluate the effectiveness of the proposed method on the Japanese-English language pair, we constructed a \emph{KU-Lecture-ST} dataset within the domain of university lectures. Additionally, leveraging some supplementary data lacking translation, we constructed another dataset \emph{KU-Lecture-ASR} for ASR pre-training.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.1
\subsection{KU-Lecture-ST}
The data for KU-Lecture-ST were collected from lectures of 39 courses delivered at Kyoto University: 6 in 2019, 27 in 2020, and 6 in 2021. 

Because raw data from 2019 were originally used for training the cascaded ST systems, they were composed of two separate parts: data for ASR training and data for MT training.
Data for ASR training include full audio recordings of the lectures, transcriptions, and timestamps that mark the beginnings and ends of the transcriptions. 
First, timestamps were obtained using speech segment detection.
Next, transcriptions of utterances defined by timestamps were obtained using a BiLSTM-HMM hybrid ASR system provided by Advanced Media, Inc.\footnote{\url{https://www.advanced-media.co.jp/english}}
Transcriptions were created using punctuation according to general linguistic conventions that determined the sentence boundaries for the dataset.
However, it should be noted that punctuation was removed in our ST experiments as is common practice for ASR.
Finally, the lecturers manually revised and confirmed the transcripts.
Data for MT training were created using the process described by Nakao et al. \citeyear{nakao2021}. This included refined transcriptions and English translations. 
Refined transcriptions (referred to as ``J整'' in Nakao et al. \citeyear{nakao2021}) are created by automatically applying simple formatting, such as the removal of fillers and the correction of phonetic contractions, to the original transcriptions (referred to as ``J話'' in Nakao et al. \citeyear{nakao2021}).
Note that refined transcriptions differ from written-style transcriptions (referred to as ``J書'' in Nakao et al. \citeyear{nakao2021}), which are converted into written language by human annotations.
Translations were made from the refined transcriptions by human translators.

Raw data from 2020 and 2021 included audio recordings, timestamps, transcriptions, and translations.
Timestamps and transcriptions were obtained using the same process as for the data from 2019.
The transcripts were translated into English by using the MT model trained by Nakao et al. \citeyear{nakao2021}.
The translation results of 4 of the 33 courses were confirmed and revised by the lecturers.

The raw data from 2019 were not directly designated for training the end-to-end ST systems, resulting in a more complex composition than the raw data from 2020 and 2021. Consequently, additional processing steps were required to prepare the 2019 data for the end-to-end ST training, as illustrated in Figure \ref{fig3}.
First, to combine the two parts of the data (ASR and MT), we conducted transcription alignment, which refers to aligning the original and refined transcriptions.
Second, sentence-level segmentation is necessary to construct this dataset because we want our model to perform ST at the sentence level; however, there are many samples with transcriptions or translations that consist of multiple sentences after transcription alignment.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F1
\begin{figure*}[b]
\begin{center}
\includegraphics{31-3ia4f1.pdf}
\end{center}
\hangcaption{Processing the data from 2019 to construct the KU-Lecture-ST dataset. Lines connect parallel data that are paired together. Bold lines indicate that they are newly paired with a certain method.} 
\label{fig3} 
\end{figure*}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.1.1
\subsubsection{Transcription Alignment}
To align the original and refined transcriptions in the 2019 data, we adopted a sentence-alignment algorithm using dynamic programming (DP) proposed by Song et al. \citeyear{DBLP:conf/lrec/SongDFK20}.
Similarity was measured using the character-level sentence BLEU-1 \cite{DBLP:conf/acl/PapineniRWZ02}.
The refined transcriptions are only used for creating data for MT training; therefore, they are not required after transcription alignment.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.1.2
\subsubsection{Sentence-level Segmentation}
We first classified the samples in the dataset into those with single-sentence transcriptions and those with multi-sentence transcriptions.
For samples with single-sentence transcriptions, we did not perform any segmentation.
For samples with multi-sentence transcriptions, we first segmented the transcriptions and translations into single sentences based on punctuation.
We then aligned the sentences based on the cosine similarity of their LASER\footnote{\url{https://github.com/facebookresearch/LASER}} embeddings with the alignment algorithm used for transcription alignment.
Finally, we generated new timestamps to align text pairs with audio using CTC segmentation \cite{DBLP:conf/specom/KurzingerWLWR20}. 
Possible misalignments were manually checked and fixed in the validation and test sets.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.2
\subsection{KU-Lecture-ASR}
For the KU-Lecture-ASR, we used raw timestamps and transcriptions from 45 courses because there were six more 2019 courses without translation. The construction process is shown in Figure \ref{fig4}. Unlike constructing the KU-Lecture-ST, we did not need to align the transcriptions and translations of the KU-Lecture-ASR. To make the dataset consistent with KU-Lecture-ST, we processed the data such that one utterance corresponded to one sentence. First, we segmented raw transcriptions using sentence boundaries and then used CTC segmentation to generate new timestamps. We then combined multiple utterances belonging to the same sentence into a single utterance. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F2
\begin{figure*}[t]
\begin{center}
\includegraphics{31-3ia4f2.pdf}
\end{center}
\hangcaption{Processing the data from 2019 to construct the KU-Lecture-ASR dataset. Lines connect parallel data that are paired together. Bold lines indicate that they are newly paired with a certain method.}
\label{fig4} 
\end{figure*}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4
\section{Method}
To integrate S2WConv into end-to-end ST, we propose a joint task of speech-to-text spoken-to-written style conversion (hereinafter referred to as \emph{SS2WConv}) and end-to-end ST, as well as two model architectures to apply multi-task learning with the joint task.

Our hypothesis is that when initializing the decoder for ST from an MT model pre-trained on bilingual corpora in written language, converting the transcription into written-style text can bridge the stylistic gap between the bilingual corpora and speech transcription, thereby enhancing the performance of end-to-end ST.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.1
\subsection{Task Definition}
Three different tasks are performed in the proposed method: ASR, ST, and SS2WConv.
The relationships among the different tasks are shown in Figure \ref{fig6}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F3
\begin{figure*}[t]
\begin{center}
\includegraphics{31-3ia4f3.pdf}
\end{center}
\hangcaption{Relations of three sub-tasks of our multi-task learning method (ASR, ST, and SS2WConv), one task for data synthesizing (S2WConv), and one task for pre-training (MT).}
\label{fig6} 
{\vskip -10pt}
\end{figure*}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\begin{itemize}

\item For ASR, the input sequence $\boldsymbol{s} = [s_1, ..., s_{T_s} ] $ is a sequence of speech features. The output sequence $\boldsymbol{x} = [x_1, ..., x_{T_x} ]$ is the corresponding transcription, where $T_x$ indicates the length of the transcription. 

\item For end-to-end ST, the input sequence $\boldsymbol{s} = [s_1, ..., s_{T_s} ] $ is the same with ASR and the output sequence $\boldsymbol{y} = [y_1, ..., y_{T_y} ]$ is the corresponding translation in the target language, where $T_y$ indicates the length of the translation.

\item For SS2WConv, the input sequence $\boldsymbol{s} = [s_1, ..., s_{T_s} ] $ is also the same with ASR and the output sequence $\boldsymbol{z} = [z_1, ..., z_{T_z} ]$ is the corresponding written-style transcription, where $T_z$ indicates the length of the written-style transcription. Written-style transcription refers to a sentence with the same semantic meaning as the transcription but is in written language instead of spoken language.

\end{itemize}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.2
\subsection{Model Architectures}
Inspired by Le et al. \citeyear{DBLP:conf/coling/LePWGSB20}, we propose our model architecture \emph{SS2WConv-ST dual-decoder Transformer }.
Correspondingly, the original dual-decoder Transformer is referred to as \emph{ASR-ST dual-decoder Transformer }, which is adopted as the baseline.

Figure \ref{fig1} shows the general structures of the baseline model and the proposed model. 
Both models are based on Transformer \cite{DBLP:conf/nips/VaswaniSPUJGKP17} and consist of one encoder and two sub-decoders. Each sub-decoder specializes in producing outputs for a specific task.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F4
\begin{figure*}[b]
{\vskip 15pt}
\begin{center}
\includegraphics{31-3ia4f4.pdf}
\end{center}
\caption{General structures of baseline (left) and the proposed model (right).}
\label{fig1} 
\end{figure*}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.2.1
\subsubsection{ASR-ST Dual-decoder Transformer}
ASR-ST dual-decoder Transformer performs a joint task of ASR and ST, taking a sequence of speech features $\boldsymbol{s}$ as input and outputting a transcription $\boldsymbol{x}$ and a translation $\boldsymbol{y}$. The output distributions can be written as follows:

{\setlength\abovedisplayskip{0pt}.
\begin{align}
    D_{asr\mathchar`-st} &= p(\boldsymbol{x}, \boldsymbol{y}|\boldsymbol{s})\notag\\
    &= \prod_{t=0}^{max(T_x, T_y)} p(x_t, y_t|I(t, T_x, T_y))
\end{align}}

where, 


\[
I(t, T_x, T_y) = 
\begin{cases} 
\boldsymbol{s} & \text{if } t = 0 \\
\boldsymbol{x}_{<t}, \boldsymbol{y}_{<t}, \boldsymbol{s} & \text{if } 0 < t \leq \min(T_x, T_y) \\
\boldsymbol{x}, \boldsymbol{y}_{<t}, \boldsymbol{s} & \text{if } T_x < t \leq T_y\\
\boldsymbol{x}_{<t}, \boldsymbol{y}, \boldsymbol{s} & \text{if } T_y < t \leq T_x 
\end{cases}
\]


The training objective is the weighted sum of the cross-entropy losses for both tasks.
\begin{equation}
\label{eqn:loss_asr}
    L_{asr\mathchar`-st} = \alpha L_{asr} + (1-\alpha) L_{st}
\end{equation}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.2.2
\subsubsection{SS2WConv-ST Dual-decoder Transformer}
SS2WConv-ST dual-decoder Transformer performs a joint task of SS2WConv and ST, taking a sequence of speech features $\boldsymbol{s}$ as input and outputting a written-style transcription $\boldsymbol{z}$ and a translation $\boldsymbol{y}$. 
Similar to the ASR-ST dual-decoder Transformer, the output distributions can be written as
{\setlength\abovedisplayskip{0pt}.
\begin{align}
    D_{ss2wconv\mathchar`-st} &= p(\boldsymbol{y}, \boldsymbol{z}|\boldsymbol{s})\notag\\
    &= \prod_{t=0}^{max(T_y, T_z)} p(y_t, z_t|I(t, T_x, T_z))
\end{align}}

where, 

\[
I(t, T_x, T_y) = 
\begin{cases} 
\boldsymbol{s} & \text{if } t = 0 \\
\boldsymbol{x}_{<t}, \boldsymbol{z}_{<t}, \boldsymbol{s} & \text{if } 0 < t \leq \min(T_x, T_z) \\
\boldsymbol{x}, \boldsymbol{z}_{<t}, \boldsymbol{s} & \text{if } T_x < t \leq T_z\\
\boldsymbol{x}_{<t}, \boldsymbol{z}, \boldsymbol{s} & \text{if } T_z < t \leq T_x 
\end{cases}
\]


The training objective is the weighted sum of the cross-entropy losses for both tasks.
\begin{align}
\label{eqn:loss_conv}
    L_{ss2wconv\mathchar`-st} &= \alpha L_{ss2wconv} + (1-\alpha) L_{st}
\end{align}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.3
\subsection{Interaction between Decoders}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F5
\begin{figure*}[b]
\begin{center}
\includegraphics{31-3ia4f5.pdf}
\end{center}
\hangcaption{Interactive attention mechanism in dual-decoder Transformers, highlighted with the dotted box. It merges the outputs of the encoder-decoder cross-attention and decoder-decoder cross-attention sub-layers.}
\label{fig5} 
\end{figure*}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


Different decoders can exchange information with each other through the interactive attention mechanism \cite{DBLP:conf/aaai/LiuZXZHWWZ20}, which refers to the addition of decoder-decoder cross-attention sub-layers in the standard Transformer decoder.
We adopted the variation that exhibited the best performance in Le et al. \citeyear{DBLP:conf/coling/LePWGSB20}, as illustrated in Figure \ref{fig5}. We added a decoder-decoder cross-attention sub-layer in addition to the encoder-decoder cross-attention sub-layer. 
The decoder-decoder cross-attention receives query $\boldsymbol{Q}$ from the output of the masked self-attention of decoder A and receives key $\boldsymbol{K}$ and value $\boldsymbol{V}$ from the output of the masked self-attention of decoder B.
The final output is obtained by merging the output of the encoder-decoder cross-attention $\boldsymbol{H}_{main}$ with the output of the decoder-decoder cross-attention $\boldsymbol{H}_{cross}$.
Linear interpolation is adopted as the merging function. Therefore, the output representation of the merged interactive attention is
{\setlength\abovedisplayskip{0pt}.
\begin{align}
    \boldsymbol{H}_{dual} &= \boldsymbol{H}_{main} + \lambda \boldsymbol{H}_{cross}
\end{align}}
where 
$\lambda$ is a learnable parameter.

For both sub-decoders, the decoding process starts simultaneously without adopting a \emph{wait-k} policy \cite{DBLP:conf/acl/MaHXZLZZHLLWW19}, which is based on the results of Le et al. \citeyear{DBLP:conf/coling/LePWGSB20}.
However, a notable difference in our study is found in the translation language pair. Japanese and English have very different sentence structures, whereas Le et al. focused on European languages, which generally share similar sentence structures.
The absence of a wait-k policy presents a challenge: the model may generate target language content that is not relevant to the source language content to which it has access during the translation. Therefore, employing the wait-k policy could be more effective for Japanese-English translation. We leave this exploration for future studies.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.4
\subsection{Synthesizing Written-style Transcriptions with S2WConv}
Although it is possible to create written-style transcriptions manually, we adopted a simpler approach: synthesizing written-style transcriptions from transcriptions with a pre-trained S2WConv model and utilizing them as the ground truth for SS2WConv in multi-task learning.
The reason for this decision is that S2WConv is not a particularly challenging task, and even a small training set can yield satisfactory performance. Our experimental results in Section 5 demonstrate that employing synthesized written-style transcriptions can improve ST performance.
Furthermore, this approach significantly simplified the training process on new datasets.

We followed Nakao et al. \citeyear{nakao2021} and adopted the LaserTagger \cite{DBLP:conf/emnlp/MalmiKRMS19} as the model architecture. The necessary data size for training a well-performing S2WConv model is significantly smaller than that for other tasks because of the similarity between transcriptions and written-style transcriptions.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5
\section{Experiments}
To verify the effectiveness of the proposed approach, ST performance was evaluated using the KU-Lecture-ST dataset. We then compared it with various baseline models, including ASR-ST dual decoders (with and without interactive attention), single decoders, and cascaded models.
A comparison was performed across multiple settings for MT pre-training to demonstrate the limitations of the proposed method. We also provide a comprehensive error analysis of the translation results from the different models.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.1
\subsection{Datasets}
We used the constructed KU-Lecture-ST dataset to evaluate the ST performance of the baseline and proposed models. 
The validation and test sets were exclusively selected from the 2019 data to avoid machine-translated data.
For ASR pre-training, we used the KU-Lecture-ASR and the 7th version of CSJ.\footnote{\url{https://ccd.ninjal.ac.jp/csj/en/}}
For MT pre-training, we used the Japanese-English section of two corpora: ASPEC \cite{DBLP:conf/lrec/NakazawaYUUSKI16} and OpenSubtitles \cite{lison-tiedemann-2016-opensubtitles2016}. 
The former is written language, and the latter is spoken language. Therefore, we can compare the performance of our models when the language style gap between the input speech and the MT dataset is significant with that when it is small.
However, the domains of the two corpora differ, as ASPEC contains data from scientific articles and OpenSubtitles contains data from the subtitles of movies and TV series. 
Therefore, we used our S2WConv model to generate a written-style version of OpenSubtitles corpora (\emph{OpenSubtitles-WR}) for further comparison. 
For S2WConv, we used the same parallel data of transcriptions and written-style transcriptions as in Nakao et al. \citeyear{nakao2021}.
This dataset is referred to as \emph{KU-Lecture-S2WConv}.
Table \ref{table1} lists the details of all the datasets used. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T2
\begin{table}[b]
\input{04table02.tex}
\hangcaption{The number of utterances in ST/ASR datasets and the number of sentences in the MT/S2WConv dataset. }
\label{table1}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.2
\subsection{S2WConv}
We trained an S2WConv model on KU-Lecture-S2WConv with the LaserTagger \cite{DBLP:conf/emnlp/MalmiKRMS19} using the same training setups described by Nakao et al. \citeyear{nakao2021}. 
The S2WConv model obtained a SARI score \cite{DBLP:journals/tacl/XuNPCC16} of $80.6$ on the test set of KU-Lecture-S2WConv.
This model was used in two aspects of our experiments: (i) To synthesize written-style transcriptions from transcriptions in the KU-Lecture-ST.
(ii) Generate OpenSubtitles-WR from the OpenSubtitles corpus.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.3
\subsection{Preprocessing and Model Settings}
The English translations were normalized and tokenized using Moses tokenizers \cite{DBLP:conf/acl/KoehnHBCFBCSMZDBCH07}. 
Japanese and written-style transcriptions were tokenized using JUMAN++ \cite{morita-etal-2015-morphological,tolmachev-etal-2018-juman}, and punctuation was stripped.
The Japanese and English tokens were further split into subwords using the BPE method \cite{DBLP:conf/acl/SennrichHB16a} with
a joint vocabulary of 16k subwords. 

We used Kaldi \cite{DBLP:conf/icassp/RavanelliPB19} to extract an 83-dimensional feature for each frame with a frame size of $25$ ms and a frame shift of $10$ ms. 
Each feature consisted of an 80-channel log Mel Filter Bank coefficient (FBank) and a 3-dimensional pitch feature.
FBank is derived from the spectral representation of audio and contains robust speech recognition characteristics. 
Pitch features are used to capture the tonal characteristics of the speech.
The extracted features were normalized using the mean and standard deviation computed on the training set to reduce the effects of channel noise and other environmental factors.
We removed utterances with more than 6000 frames or 400 characters.
We used speed perturbation \cite{DBLP:conf/acl/InagumaKDKYHW20} with factors of 0.9, 1.0, and 1.1 for data augmentation.

Our implementation is based on the ESPnet-ST toolkit  \cite{DBLP:conf/acl/InagumaKDKYHW20}. 
We used the same architecture for all ST models, with a $12$-layer encoder and $8$-layer decoders. 
All Transformer layers adopted 2048-dimensional feed-forward networks and 4-headed 256-dimensional attention sub-layers.

All encoders in the ST models were initialized with an ASR model trained first on CSJ and then on KU-Lecture-ASR. 
ST decoders were either trained from scratch or initialized with an MT model pre-trained on ASPEC, OpenSubtitles, or OpenSubtitles-WR. 
ASR and SS2WConv decoders are trained from scratch.
We used a Transformer with a $12$-layer encoder and a $6$-layer decoder for ASR pre-training and a Transformer with a $6$-layer encoder and an $8$-layer decoder for MT training. 
Checkpoints in ASR/MT pre-training were selected based on the accuracy on the validation set and then used to initialize ST training.

We also conducted experiments with single-decoder end-to-end ST and cascaded ST. For single-decoder end-to-end ST, we used the single-decoder Transformer architecture with the same model setting and pre-training for the encoder and the decoder as the dual-decoder Transformer. For cascaded ST, the ASR and MT models were obtained by fine-tuning the same pre-trained models used in end-to-end ST (the one trained on ASPEC was used for MT) on the ST datasets. Between ASR and MT, a Punctuator2\footnote{\url{https://github.com/ottokart/punctuator2}} model trained on transcriptions in the KU-Lecture-ASR was used to restore punctuation. Experiments were conducted with and without S2WConv after punctuation was restored. 


We followed the settings of the dual-decoder Transformer and set $\alpha$ in the loss functions (Equations \ref{eqn:loss_asr} and \ref{eqn:loss_conv}) to $0.3$.
We used the Adam optimizer \cite{DBLP:journals/corr/KingmaB14} and the Noam learning rate schedule \cite{DBLP:conf/nips/VaswaniSPUJGKP17} with $10$k warm-up steps and a maximum learning rate of $1.5e-3$.
We used a batch size of $24$ per GPU and trained the models on a single machine using $8$ Geforce GTX $1080$ Ti GPUs.
Subsequently, the models were trained for $25$ epochs. We maintained checkpoints after each epoch, averaged the $5$ best models on the validation set based on BLEU, and used the averaged model for testing.
For decoding, the beam size was set to $5$ for the ST task and $1$ for the other tasks.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.4
\subsection{Results}
The translation results were evaluated using SacreBLEU \cite{DBLP:conf/wmt/Post18} in Table \ref{table2}.  Statistical significance tests were conducted using the bootstrap resampling method proposed by Koehn \citeyear{koehn-2004-statistical}. In addition to evaluating all ST models, we used the MT results as a performance benchmark that can serve as an upper bound for ST performance. To establish this, we utilized MT models within our cascaded systems to generate translations from ground-truth transcriptions or synthesized written-style transcriptions. For the dual-decoder models, we obtained the Character Error Rate (CER) for evaluating the performance of ASR or SS2WConv. The references were transcriptions for ASR and written transcriptions for SS2WConv.
Furthermore, we analyzed the translation results for different model settings using the Translation Error Rate (TER) \cite{DBLP:conf/amta/SnoverDSMM06}, which considers different types of translation errors. The detailed statistics are shown in Table \ref{table4}. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T3
\begin{table}[t]
\input{04table03.tex}
\hangcaption{SacreBLEU scores on KU-Lecture-ST. 
CER was used to evaluate the ASR or SS2WConv tasks in dual-decoder models.
MT results were obtained by utilizing MT models within the cascaded systems to generate translations from the ground-truth transcriptions or the synthesized written-style transcriptions. 
``(IA)'' indicates that the interactive attention mechanism is activated.
Datasets used for MT pre-training are shown in the MT pre-training column, and ``---'' indicates that there is no MT pre-training involved. 
For end-to-end models, it was used to initialize the ST decoder. For cascaded models, it was used to initialize the MT module (both the encoder and the decoder).
``$\dag$'' indicates that the result is significantly better than No. $1$, No. $5$, No. $9$, No. $13$, and No. $17$ at $p < 0.05$).
}
\label{table2}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T4
\begin{table}[t]
\input{04table04.tex}
\hangcaption{TER and detailed statistics include insertion (Ins), deletion (Del), substitution (Sub), shift (Shft), and number of words (NumWd). }
\label{table4}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.4.1
\subsubsection{SacreBLEU and CER Comparison}
We start by comparing the results of SS2WConv-S and ASR-ST with the interactive attention mechanism activated.
When using ASPEC for MT pre-training (No. $17$ and $21$), the proposed model (SS2WConv-ST) outperforms the baseline (ASR-ST) significantly. 
Conversely, ASR-ST models are relatively better when MT pre-training is not involved (No. $20$ and $24$), or conducted with spoken language (No. $18$ and $22$). This observation aligns with our hypothesis that the proposed model improves performance by reducing the gap between the language style of transcriptions and the data used in MT pre-training. Consequently, in scenarios in which there is no language-style gap, no improvement is observed.
The fact that SS2WConv-ST performs worse than ASR-ST can be attributed to the insufficient quality of the synthesized written transcription.
When MT pre-training is switched from OpenSubtitles (No. $18$ and $22$) to OpenSubtitles-WR (No. $19$ and $23$), the performance of ASR-ST declines while that of SS2WConv-ST does not, demonstrating that the proposed method benefits from written-style MT pre-training data.
SS2WConv-ST models produce a higher CER than ASR-ST models, which is reasonable because SS2WConv is a more difficult task than ASR. 
CER varies slightly across different MT pre-training settings, suggesting that ASR/SS2WConv may have reached a performance plateau with adequate ASR pre-training. One exception is observed in No. $24$, which can serve as an example of poor ST performance impacting SS2WConv through the interactive attention mechanism.

Next, we perform comparisons with other end-to-end baselines, including dual-decoder models without interactive attention activated and single-decoder models.
The models with interactive attention (No. $17$--$24$) generally yield a better ST performance than those without it (No. $13$--$16$), which highlights the effectiveness of the interactive attention mechanism.
However, such improvements are not observed in ASR performance. This suggests that the performance enhancement from interactive attention is asymmetric in our experiments, which is reasonable given that the ST task is more difficult and naturally requires support from ASR.
Without interactive attention, the dual-decoder architecture (No. $13$--$16$) fails to outperform the single-decoder architecture (No. $9$--$12$). This 
observation shows that simply adding a decoder for multi-task learning is not effective in our experimental setup.

Finally, we examine the results of the cascaded models.
The end-to-end models (No. $9$--$24$) demonstrate their advantages by outperforming their cascaded counterparts (No. $1$--$8$).
However, in prior experiments conducted with a smaller dataset (with approximately 71K training utterances), the results for the cascaded models are notably better than those for the end-to-end models (by approximately $4$ points in SacreBLEU). Therefore, it is crucial to note that sufficient data for ST is necessary to achieve this level of performance for end-to-end ST.
When comparing the cascaded models with S2WConv as an intermediate task (No. $5$--$8$) and those without (No. $1$--$4$), the former exhibits better performance when trained with ASPEC but not with other MT pre-training data. This trend mirrors the comparison between SS2WConv and ASR-ST dual-decoder models, illustrating a similar relationship between the conversion tasks and MT pre-training in both cascaded and end-to-end settings.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.4.2
\subsubsection{Error Analysis}
The statistics in Table \ref{table4} show a distinct trade-off between deletion (Del) and insertion/{\linebreak}substitution (Ins/Sub) errors.
A high Ins/Sub combined with a low Del typically indicates an elaborate but verbose translation. These translations contain additional words or phrases that may not have been included in the references.
Conversely, a high Del combined with a low Ins/Sub suggests a concise but incomplete translation. These translations often skipped certain words or phrases presented in the references.

The single-decoder E2E models (No. $9$--$12$), in which there is no guidance from other tasks, tend to produce verbose translations. Multi-task learning with ASR (No. $13$--$24$) dramatically alters the translation style, pushing it more concisely. This suggests that multi-task learning in dual-decoder models acts as a constraint on the ST decoders.


The SS2WConv-ST models (No. $21$--$24$), compared to ASR-ST (No. $17$--$20$), tend to produce translations that are more on the elaborate/verbose side.  
When comparing using the written-style MT pre-training data (No. $17$, $21$, $19$, $23$) to using spoken-style ones (No. $18$, $22$), the translations are more concise, in contrast to the verbosity introduced by SS2WConv-ST. 
By examining all the results, we can observe that the combination of SS2WConv-ST and written-style MT pre-training achieves the best trade-off, generating limited errors in all error categories.
The example outputs presented in Table \ref{table5} demonstrate how this error tendency manifests in translation prediction.
In both examples, the following observations can be made: The ASR-ST model (No. $17$) produces more Del errors than the SS2WConv-ST model (No. $21$), spoken-style MT pre-training (No. $22$) produces more Ins errors than written-style MT pre-training (No. $21$). The combination of SS2WConv-ST and written-style MT pre-training (No. $21$) generates the translation with the lowest total number of errors.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T5
\begin{table}[t]
\input{04table05.tex}
\hangcaption{Examples of translation predictions of 3 different models (No. $17$, $22$ and $21$). Statistics of different error types are shown in the brackets at the end of predictions. }
\label{table5}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6
\section{Conclusion}
In this study, we propose a joint task of SS2WConv and end-to-end ST, and an interactive attention-based multi-decoder architecture to perform this task. Compared with training on the joint task of ASR and ST, our method reduced the gap between the language styles of speech transcription and the bilingual corpora used for MT pre-training. Experiments on Japanese-English ST datasets illustrated the effectiveness of our method. 
In the future, we plan to extend this method to other language pairs.



% Start Acknowledgment
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment
We would like to thank Advanced Media, Inc., for their cooperation in dataset construction.


% Start bibliography
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.7}
    \bibliography{04refs}



% Start biography
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Zhengdong Yang}{%
Zhengdong Yang received his B.S. in Software Engineering from Beijing University of Technology in 2019 and his M.S. in Informatics from Kyoto University in 2023.
He is currently a Ph.D. student at Kyoto University. His research interests include natural language processing, speech translation, and automatic speech recognition.
}

\bioauthor[:]{Shuichiro Shimizu}{%
Shuichiro Shimizu received his B.S. in Electrical and Electronic Engineering and M.S. in Informatics from Kyoto University in 2021 and 2023, respectively.
He is currently a Ph.D. student at Kyoto University. His research interests are natural language processing covering speech, especially applications related to dialogue and machine translation.
}

\bioauthor[:]{Chenhui Chu}{%
Chenhui Chu received his B.S. in Software Engineering from 
Chongqing University in 2008, and his M.S. and Ph.D. in Informatics 
from Kyoto University in 2012 and 2015, respectively. He is currently 
a program-specific associate professor at Kyoto University. His 
research interests include natural language processing, specifically 
machine translation and multimodal machine learning. 
}

\bioauthor[:]{Sheng Li}{%
Sheng Li received his B.S. and M.E. degrees in 2006 and 2009 from Nanjing University, Nanjing, China, and his Ph.D. from Kyoto University, Kyoto, Japan, in 2016. He is currently with the National Institute of Information and Communications Technology (NICT), Kyoto, Japan, as a researcher working on speech recognition. He is a member of the International Speech Communication Association (ISCA) and IEEE. 
}

\bioauthor[:]{Sadao Kurohashi}{%
Sadao Kurohashi received a PhD in Electrical Engineering from Kyoto University in 1994. 
He is currently the Director-General of the National Institute of Informatics, Japan, and a specially appointed professor at the Graduate School of Informatics at Kyoto University.
His research interests include natural language processing, knowledge infrastructure, and open science.
}
\end{biography}

\biodate   %%%% Following settings will be edited by editors.


\end{document}


