\documentclass[english]{jnlp_3.8}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{array}
\allowdisplaybreaks

\usepackage{hangcaption_jnlp}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition

%%%%%\usepackage{color}
\usepackage{enumerate}
\usepackage{booktabs}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{31}
\Number{3}
\Month{September}
\Year{2024}
%%%%\Issuetitle{「　　　　」特集号}

\received{2023}{12}{1}
\revised{2024}{3}{1}
\accepted{2024}{4}{3}



\setcounter{page}{868}



%%%%   Title, Author, etc.            %%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{NAIST Simultaneous Interpretation Corpus: Development and Analyses of Data from Interpreters of Different Levels}
\eauthor{Kosuke Doi\affiref{Author_1} \and Katsuhito Sudoh\affiref{Author_1}\affiref{Author_2} \and Satoshi Nakamura\affiref{Author_1}\affiref{Author_3}} 
\eabstract{%
This paper describes the development of a large-scale English-Japanese simultaneous interpretation corpus named NAIST-SIC and presents analyses of it. We collected the recordings of simultaneous interpreting sentences (SI$^\text{sent}$). To understand the characteristics of simultaneous interpreting by human simultaneous interpreters (SIers), we analyzed a subset of this corpus. Samples of speech were interpreted by three SIers having different levels of experience and can be used to compare SI$^\text{sent}$ attributes in terms of the SIers' experience. Using this corpus subset, we analyzed the differences in latency, quality, and word order. The results show that (1) SIers with more experience tended to generate a higher quality of SI$^\text{sent}$, and (2) they better controlled the latency and quality. We also observed that (3) a large latency degraded the SI$^\text{sent}$ quality.
}
\ekeywords{Simultaneous Interpretation Corpus, Quality, Latency, Word Order}

\headauthor{Doi et al.}
\headtitle{NAIST SIC: Development and Analyses of Data from Interpreters of Different Levels}


%%%%     Affiliation     %%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\affilabel{Author_1}{}{Nara Institute of Science and Technology}
\affilabel{Author_2}{}{Work conducted at Nara Institute of Science and Technology. Present affiliation: Nara Women's University}
\affilabel{Author_3}{}{The Chinese University of Hong Kong, Shenzhen}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S1
\section{Introduction}
\renewcommand{\thefootnote}{}
\footnote[0]{Part of this work was presented at the 18th International Conference on Spoken Language Translation \cite{doi-etal-2021-large} and the 23rd Annual Conference of the Japan Association for Interpreting and Translation Studies \cite{doi-etal-2022-jaits}}
\renewcommand{\thefootnote}{\arabic{footnote}}
With recent developments in deep-learning technologies, many machine translation services have become available, although they still have various problems.
Certain services provide automatic speech-to-speech translation \cite{nakamura-book-2018,sumita-2022}.
However, these need to wait for the completion of input to start the translation.
Recent studies on automatic simultaneous interpreting (SI) \cite<e.g.,>{zheng-etal-2019-simultaneous,ko-etal-2023-tagged} have aimed to overcome this limitation.
While an automatic SI system should learn how human simultaneous interpreters (SIers) achieve SI, the amount of simultaneous interpretation corpora (SICs) for learning what human SIers do remains limited and insufficient for system development.
Most of the recent automatic SI studies use speech translation corpora \cite<e.g., MuST-C;>{di-gangi-etal-2019-must} whose translations are generated based on completed audio data or transcripts, even though SI is different in characteristic from consecutive interpreting (CI) or written translation (WT).

We have been constructing a large-scale English-Japanese SIC, named NAIST Simultaneous Interpretation Corpus (NAIST-SIC),\footnote{A part of the corpus is available at \url{https://dsc-nlp.naist.jp/data/NAIST-SIC/}.} for the purposes of analyzing human simultaneous interpreting sentences (SI$^\text{sent}$) and developing automatic SI systems.
We recorded SI$^\text{sent}$ made by professional SIers. 
The total recording time in both the directions (i.e., English to Japanese and vice versa) was over 300 hours.
Samples of speech were interpreted by three SIers with different levels of experience, following the design of \citeA{shimizu-etal-2014-collection}.
This corpus subset, denoted as SIC$^\text{comparable}$, enables us to compare the SI$^\text{sent}$ differences in terms of the SIers' experience.
In addition, written translation sentences (WT$^\text{sent}$) are available for certain parts of the data, including SIC$^\text{comparable}$.
Given these features, NAIST-SIC could also be a useful resource for interpretation and translation studies.

This paper describes the development of NAIST-SIC and presents the results of our analyses.
We analyzed some SI$^\text{sent}$ within SIC$^\text{comparable}$ to compare the differences in latency, quality, and word order.
While SI$^\text{sent}$ was compared with WT$^\text{sent}$ in our previous studies \cite{doi-etal-2021-large,doi-etal-2022-jaits}, we conducted further analyses against source speeches in this paper.
Analyzing the correspondence between source sentences and SI$^\text{sent}$ would help us understand what SIers actually do and develop automatic SI systems.
We investigated which contents in the source speeches were expressed in what order and whether there were differences between SI$^\text{sent}$ and WT$^\text{sent}$.
We observed that experienced SIers tended to generate higher-quality SI$^\text{sent}$ while maintaining the word order of the source language.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2
\section{Related Work}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2.1
\subsection{Existing Simultaneous Interpretation Corpora}
Despite their usefulness, the development of SICs requires considerable effort and caution.
Consequently, the number of SICs is limited.
Table \ref{table:simul-corpora} lists the publicly available English-Japanese SICs.
The Simultaneous Interpretation Database \cite<SIDB,>{tohyama-et-al-2004} consists of over 180 hours of recordings including monologues (lectures) and dialogues (travel conversations).
The corpus of \citeA{shimizu-etal-2014-collection} is relatively small (22 hours) and has two significant features: (1) all of the speeches have SI$^\text{sent}$ from three interpreters with different levels of experience, and (2) WT$^\text{sent}$ are available for some of the speeches.
These features enable comparisons among the SI$^\text{sent}$ themselves as well as with the WT$^\text{sent}$.
The JNPC Corpus \cite{matsushita-et-al-2020} consists of videos of press conferences, along with their transcripts.
Time information is annotated on every word for English and on every tokenized segment or morpheme for Japanese.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T1
\begin{table}[t]
\input{02table01.tex}
\caption{Existing English-Japanese simultaneous interpretation corpora and our corpus}
\label{table:simul-corpora}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


In language pairs other than English-Japanese, \citeA{Paulik-and-Waibel-2009} developed an SI system using 207 hours of SI$^\text{sent}$ collected from the European Parliament Plenary Sessions.
In the 2020s, several datasets have been released one after another, some of which are relatively large-scale \cite<e.g.,>[17.3~K hours, 15 languages]{wang-etal-2021-voxpopuli}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2.2
\subsection{Simultaneous Interpreting}
SI is the task of translating speech from a source language into a target language in real time.
SI imposes a high cognitive load, since SIers need to accomplish multiple tasks (e.g., listening, understanding, translating, and speaking) at the same time \cite{mizuno-2016,ryu-2018}.
To effectively allocate limited attention or processing capacity, SIers use sophisticated strategies including segmentation, summarization, and generalization \cite{he-etal-2016-interpretese}.
SIers also adopt the strategy of maintaining word order in the source language to shorten the delay and reduce the working memory load \cite{cai-2020-features}.
As a result, SIers produce translations with low latency, and phrases that do not affect the main idea are not necessarily translated.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2.3
\subsection{Analyses of Interpreting Sentences}
The characteristics of interpreting sentences have typically been investigated from the perspectives of latency, quality, and word order.
\citeA{ohara-2003} analyzed cross-language conversations through SI and CI, finding that SI made the conversations more time-efficient and smooth.
For evaluating the latency by human SIers, Ear-Voice Span (EVS) is a widely used metric to denote the lag between the original utterances and corresponding SI$^\text{sent}$.
The EVS of SI$^\text{sent}$ is known to be about two to four seconds \cite<see>{Robbe-2019}.

Quality analyses generally rely on manual evaluations of corpus data.
\citeA{ino-2008} compared the faithfulness between SI$^\text{sent}$ and prepared interpreting sentences.
Their analyses of actual interpreting sentences revealed that the SI$^\text{sent}$ was comparable in quality to the prepared interpreting sentences despite the time constraints and cognitive load.
\citeA{Shinzaki-2017} compared evaluations of content, delivery, and faithfulness by different listeners between SI$^\text{sent}$ and consecutive interpreting sentences (CI$^\text{sent}$).
General listeners rated CI$^\text{sent}$ higher in all the aspects, whereas professional interpreters rated SI$^\text{sent}$ higher in delivery.
\citeA{dong-2018} compared the quantities of contents in a source speech that were translated in SI$^\text{sent}$ and CI$^\text{sent}$ using speeches of different difficulty levels.
The ratios of translated content were lower for difficult source speeches in both the interpretation modes, whereas CI$^\text{sent}$ retained more content than SI$^\text{sent}$.

Word order has also been studied intensively.
\citeA{cai-2020-affects} statistically analyzed word order differences between WT$^\text{sent}$ and SI$^\text{sent}$, and they identified both syntactic and non-syntactic factors that affect the word order decision.
\citeA{okamura-2023} quantitatively analyzed the degrees to which the word order of the source text was preserved, and they observed that SI$^\text{sent}$ retained the order better than CI$^\text{sent}$ or WT$^\text{sent}$.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3
\section{Corpus Development}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.1
\subsection{Materials}
\label{ssec:Materials}
NAIST-SIC was compiled from four types of materials.
For the English-to-Japanese direction, TED talks were used.
For the Japanese-to-English direction, speeches from TEDx, CSJ, and JNPC were used.

\textbf{TED}: TED\footnote{\url{https://www.ted.com/about/conferences}} provides short talks on various topics from science to culture.
Videos of these talks are available on its website.
TED talks have been manually transcribed and translated by volunteers, and Japanese WT$^\text{sent}$ (i.e., subtitles) are available for many talks.

\textbf{TEDx}: TEDx\footnote{\url{https://www.ted.com/about/programs-initiatives/tedx-program}} is an event where local speakers present topics to local audiences.
The events are held under a license granted by TED.
The talks follow the format of TED talks.
The videos are available on YouTube and the TED website.

\textbf{CSJ}: The Corpus of Spontaneous Japanese \cite{maekawa-2003} is a spoken corpus whose main sources are academic presentations and speeches on everyday topics.
It contains audio data and their transcripts with linguistic annotations.

\textbf{JNPC}: The Japan National Press Club organizes press conferences featuring Japanese and foreign guest speakers from politicians to business representatives.
These are video-recorded and available online, with transcripts for some of these provided on the website.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.2
\subsection{Recording}
Professional SIers with different levels of experience interpreted speeches from the materials described in Section \ref{ssec:Materials}.
Each SIer was assigned a rank based on their length of experience, as in \citeA{shimizu-etal-2014-collection} (Table \ref{table:interpreter-rank}).
The recordings were made from 2018 to 2020.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T2
\begin{table}[b]
\input{02table02.tex}
\caption{Ranks of simultaneous interpreters}
\label{table:interpreter-rank}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


SIers wore a headset and interpreted speech while watching video on a computer.
They only listened to the audio while interpreting the CSJ speech because no video was available.
Each speech was interpreted by one SIer without a break because most of our sources were shorter than 15 minutes\footnote{The average talk lengths of materials: TED 11.20 minutes, TEDx 15.85 minutes, CSJ 13.55 minutes, and JNPC 84.33 minutes. A long speech was divided to be interpreted by the same interpreter after breaks.}.
The SIers were provided with documents related to speech in advance to improve the SI$^\text{sent}$ quality.
In fact, related information or materials (e.g., presentation slides) are usually provided to them in their actual daily work.
The following list provides details of the documents given prior to our recording sessions:
\begin{itemize}
    \item TED, TEDx (2018): Summary of talks; referenceable during SI
    \item TED (2019--): English transcripts from TED website; not referenceable during SI
    \item TEDx (2019--): Japanese subtitles generated by YouTube; not referenceable during SI
    \item CSJ: 10\% summary of Japanese transcripts; referenceable during SI
    \item JNPC: No documents provided.
\end{itemize}

Table \ref{table:recorded-hours} gives a breakdown of the recorded hours of NAIST-SIC.
A total of four hours of TED and TEDx recorded in 2018 were interpreted by SIers of all the three ranks (i.e., SIC$^\text{comparable}$, 4 hours $\times$ 3 SIers $=$ 12 hours; indicated with asterisks).
The other talks were interpreted by either an S- or A-rank SIer.
Almost all the recorded SI$^\text{sent}$ were manually transcribed\footnote{Some Japanese-to-English SI$^\text{sent}$ data remain untranscribed.}.
The entire corpus consists of SI$^\text{sent}$ with over 1,200 talks.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T3
\begin{table}[t]
\input{02table03.tex}
\caption{Recorded hours of NAIST-SIC. The figures with asterisk (*) indicate SIC$^\text{comparable}$.}
\label{table:recorded-hours}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4
\section{Corpus Analyses}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.1
\subsection{Data}
To understand the characteristics of SI by human SIers, analyses were performed based on the aspects of latency, quality, and word order using our new corpus.
We selected English-to-Japanese SI$^\text{sent}$ data from 14 TED talks and examined SI$^\text{sent}$ differences among the interpreter ranks, as well as the differences between SI$^\text{sent}$ and WT$^\text{sent}$.
The talks were a subset of SIC$^\text{comparable}$, which is the SI$^\text{sent}$ that SIers of all the three ranks generated from the same source speeches (see Tables \ref{table:interpreter-rank} and \ref{table:recorded-hours}).

The SI$^\text{sent}$ transcriptions were made by human annotators based on segments as shown in Figure \ref{example-transcript}.
Each segment was annotated with an ID and start/end timestamps in milliseconds.
Here, a segment does not necessarily correspond to an individual sentence.
For SI$^\text{sent}$, discourse tags (e.g., fillers, slips of the tongue, and pauses) were also annotated.
In addition to SI$^\text{sent}$, WT$^\text{sent}$ (i.e., Japanese subtitles) were used in the quality and word order analyses to examine the differences from SI$^\text{sent}$.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F1
\begin{figure}[t]
\begin{center}
\includegraphics{31-3ia2f1.pdf}
\end{center}
\hangcaption{Example of a transcript. Before each segment, IDs and start/end timestamps are annotated. Discourse tags are also used: F: filler, (?): unintelligible, $\langle$H$\rangle$: prolongation.}
\label{example-transcript}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.2
\subsection{Annotations}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.2.1
\subsubsection{Sentence-level Alignment}
\label{ssec:Sentence-level-Alignment}
SI$^\text{sent}$ and WT$^\text{sent}$ were aligned manually at the sentence level with the source speeches for a fair comparison.
Because the segments in the SI$^\text{sent}$ transcripts were based on SIers' utterances, these did not necessarily match among the SIers.
Therefore, we first segmented the English transcripts according to the following rules:

\begin{itemize}
    \item segments ending with a period (.) or a period + a double quotation mark (.'')
    \item segments ending with a question mark (?) or a question mark + a double quotation mark (?'')
    \item segments ending with a closing parenthesis.
\end{itemize}

\noindent If sentence boundaries (the first two patterns above) were observed in the middle of a segment, the segment was divided at the position, and the end/start times of the divided segments were indicated with XXXXX because the annotations were performed based on transcribed text without using the original audio.

Then, the Japanese segments were aligned to English sentences based on the following rules\footnote{Subjectively judged by the authors, except for the boundaries of the English sentences.}:

\begin{itemize}
    \item Words/phrases that are not interpreted: ignored
    \item Sentences that are not interpreted: indicated as \texttt{\small\_\_drop\_\_} in Japanese segments
    \item Sentences that are not interpreted intentionally: indicated as \texttt{\small\_\_skip\_\_} in Japanese segments (e.g., Thank you.)
    \item Sentences that do not need to be interpreted: indicated as \texttt{\small\_\_null\_\_} in Japanese segments (e.g., (Laughter))
    \item No corresponding English sentence: add \texttt{\small\_\_null\_\_} to English segments
    \item Japanese segments that correspond to multiple English sentences: divide at position corresponding to a boundary of English sentences; indicated with XXXXX for the end/start times of Japanese segments.
\end{itemize}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T4
\begin{table}[p]
\footnotesize
\centering
\input{02table04.tex}
\caption{Examples of sentence-level alignment. Sentences are delimited by a blank line.}
\label{example-sentence-alignment}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\noindent By applying these rules to a source speech and its corresponding translation, the numbers of English and Japanese segments become equal.

Examples of the data aligned at the sentence level are presented in Table \ref{example-sentence-alignment}.
In the second sentence of Example 1a, the word ``You'' and phrase ``not like here'' were not interpreted.
However, no special tag was assigned.
In contrast, a sentence that was not interpreted was assigned \texttt{\small\_\_drop\_\_}, as in Example 2.
If it appeared that a sentence was not interpreted intentionally, \texttt{\small\_\_skip\_\_} was assigned (Example 3).
In fact, the sentence ``I hear you ask.'' was not interpreted by any of the three SIers.
This suggests that SIers identified less important sentences and skipped them to follow the source speech.
\texttt{\small\_\_null\_\_} in English segments typically appeared with \texttt{\small\_\_drop\_\_} (Example 4a).
In this example, the SIer stated something related to speech content.
However, it was not stated by the original speaker.
\texttt{\small\_\_null\_\_} was also assigned in cases of mistranslation.
However, if the mistranslation is considered \textit{local} rather than \textit{global} (see Example 1b: although the phrase ``10 francs'' was incorrectly interpreted as ``二フラン [two francs]'', the overall structure of the sentence was captured correctly), the Japanese segment was aligned to the English segment.
Another example of \texttt{\small\_\_null\_\_} in an English segment is shown in Example 4b.
Here, SIer added something irrelevant to the speech content.
An example of \texttt{\small\_\_null\_\_} in the Japanese segment is also shown in Example 4b.
``(Laughter)'' illustrates that the audience was laughing and that it is not an utterance by the speaker.

In addition to the manually aligned data, we used automatically aligned data called {\linebreak}NAIST-SIC-Aligned \cite{zhao-2023-naistsicaligned}.
We used six talks from it (its subset overlapped with SIC$^\text{comparable}$) for our latency analyses.
The alignment of these six talks was based on the SI$^\text{sent}$ by S-rank SIers.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.2.2
\subsubsection{Chunk-level Alignment}
\label{ssec:chunk-level-alignment}
To examine word order differences, we divided the source sentences into chunks and aligned the corresponding translations in the target sentences to them.
When the structures of the source and target language sentences are similar, the word order differences can be analyzed based on word-level alignment \cite{carl-2016}.
However, the sentence structures of English and Japanese (the language pair analyzed in this study) differ significantly (SVO and head-initial vs. SOV and head-final).
Therefore, following a previous study \cite{okamura-2023}, we conducted analyses based on chunks.

English sentences were divided into chunks following four rules proposed by \citeA{okamura-2023}, with the addition of our own Rule 5:

\begin{enumerate}[Rule 1:]
\setlength{\leftskip}{0.5cm}
    \item Before a conjunction introducing a clause or a relative pronoun (except for one modifying a subject)
    \item Before a to-infinitive, a preposition, or a gerund followed by three or more words
    \item After a long subject (i.e., three or more words) 
    \item Before or after a comma (except one used for listing in a series of words), a semicolon, a hyphen, or other marks of punctuation 
    \item After an adverbial phrase or a prepositional phrase at the beginning of a sentence.
\end{enumerate}

Chunking was performed by the first author on the sentences sampled from three talks.
We randomly sampled 20\% of the sentences from each talk (totally 88 sentences) so that an approximately equal number of sentences came from the beginning, middle, and end of the talk.
The three talks were selected based on the following procedure:

\begin{itemize}
    \item Assign a score of 1-3 to the SI$^\text{sent}$ data (14 talks $\times$ 3 SIers) based on overall quality,
    \item Calculate the average of the scores for each talk, and assign a label of \texttt{\small high}, \texttt{\small mid}, or \texttt{\small low},
    \item Select one talk from each label.
\end{itemize}

Talks labeled \texttt{\small high} are those that are easy to interpret, whereas the talks labeled \texttt{\small low} are difficult to interpret.
We selected three talks, AlexanderWagner\_2016X (\texttt{\small Ale}), NickBostrom\_2015 (\texttt{\small Nic}), and LaurelBraitman\_2014S (\texttt{\small Lau}), for easy, medium, and difficult levels.

To check the reliability of the chunking, we asked a graduate student, who is studying the evaluation of interpretations and translations, to perform chunking on the same data.
Following previous studies \cite{grouin-etal-2011-proposal,hripcsak-2005}, we used precision, recall, and F1 score as the agreement metrics.
The scores were calculated while regarding the annotation by the first author as the reference.

The results demonstrate a relatively high inter-annotator agreement (F1 $= 0.74$; see Table \ref{table:inter-annotator-reliability}).
High precision with low recall suggests that the graduate student divided sentences less frequently than did the first author.
An analysis of the annotation results show that the graduate student tended to neglect Rule 3.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T5
\begin{table}[b]
\input{02table05.tex}
 \caption{Inter-annotator reliability for chunk annotations}
 \label{table:inter-annotator-reliability}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


Japanese sentences were automatically divided into \emph{bunsetsus}\footnote{A \emph{bunsetsu} is a basic unit of dependency in Japanese that consists of one or more content words and the following zero or more function words \cite{kawahara-kurohashi-2006-fully}.} using the Juman++ Japanese morphological analyzer Ver. 1.02 \cite{morita-etal-2015-morphological} and the KNP parser Ver. 4.19 \cite{kawahara-kurohashi-2006-fully}.
Disfluencies in the SI$^\text{sent}$ were removed with the help of discourse tags before applying Juman++ and KNP.

Japanese \emph{bunsetsus} were manually aligned to corresponding English chunks.
An example of a chunk-level alignment is presented in Table \ref{table:chunk-level-alignment}.
English chunks divided based on the above rules may contain more than one content word, whereas \emph{bunsetsus} divided using the above tools typically contain one content word.
Consequently, more than one \emph{bunsetsu} can be aligned to an English chunk.
To examine the order of translation based on the chunks, as recommended in previous studies \cite{mizuno-2016,okamura-2023}, reordering within a chunk was ignored in the chunk-level alignment process (see the example in Table \ref{table:chunk-level-alignment}).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T6
\begin{table}[t]
\input{02table06.tex}
\hangcaption{Example of a chunk-level alignment. The numbers preceding Japanese \emph{bunsetsus} represent the order of appearance in the SI$^\text{sent}$. Reordering within a chunk (e.g., 1 少し [for a minute] / 2 診断に [diagnosis] / 3 ついて [about]) is ignored.}
\label{table:chunk-level-alignment}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


First, the first author and the above graduate student performed alignment on 25\% of the data.
Since a high inter-annotator agreement was achieved (Table \ref{table:inter-annotator-reliability}), the remainder of the data were annotated by the first author.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.2.3
\subsubsection{Phrase-level Alignment}
\label{ssec:phrase-level-alignment}
In a chunk-level alignment, it is unclear which elements in a chunk are translated.
To examine the differences in quality between SI$^\text{sent}$ and WT$^\text{sent}$, we needed a finer level of alignment.
Therefore, phrase-level alignment was performed.
We selected phrase-level rather than word-level so that noun phrases (e.g., ``a minute,'' ``a survey measure'') would not be split.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F2
\begin{figure}[t]
\begin{center}
\includegraphics{31-3ia2f2.pdf}
\end{center}
\caption{Example of phrase-level alignment}
\label{example-phrase-alignment}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


With the help of labels\footnote{\url{https://gist.github.com/nlothian/9240750}} assigned by Stanza \cite{qi-etal-2020-stanza}, an automatic syntactic analyzer, we divided English sentences into phrases before clause-level tags, phrase-level tags, and coordinating conjunctions (CC tags).
Then, Japanese \emph{bunsetsus} were manually aligned with English phrases by the first author.

Figure \ref{example-phrase-alignment} shows an example of this alignment.
If a \emph{bunsetsu} corresponded to both a function word and a content word, it was aligned to the content word (\textit{少し} was aligned to \emph{a minute} but not to \emph{for}).
If a \emph{bunsetsu} corresponded to multiple content words, it was aligned to all of those content words (\textit{見ましょう} was aligned to both \emph{let} and \emph{'s}).
The content words were defined as nouns, verbs, adjectives, and adverbs, following \citeA{fromkin-2011}.

Similar to \citeA{ino-2008}, we assigned a point to each alignment.
If the meaning of an English phrase was conveyed successfully, it received one point.
If the meaning was conveyed partially, half a point was awarded.
Phrase-level alignment was performed on 20\% of the sentence samples, as described in Section \ref{ssec:chunk-level-alignment}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.3
\subsection{Overall Trends}
Table \ref{table:corpus-stats} provides the basic statistics for the WT$^\text{sent}$ and SI$^\text{sent}$ of the 14 TED talks.
WT$^\text{sent}$ had a larger number of \emph{bunsetsus} than SI$^\text{sent}$, which suggests that SIers did not interpret all of the contents of the source speeches (i.e., they used strategies such as summarization).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T7
\begin{table}[b]
\input{02table07.tex}
\caption{Comparison of WT$^\text{sent}$ data and SI$^\text{sent}$ data}
\label{table:corpus-stats}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



Among SIers, B-rank SIers produced the longest SI$^\text{sent}$ (see \texttt{\small \# bunsetsu per talk} in Table \ref{table:corpus-stats}).
However, they frequently added something related to the speech content that was not stated by the original speaker and mistranslated source sentences (see \texttt{\small en null}).
The ratio of \texttt{\small en null} decreased as the amount of experience became longer.
In addition, the ratio of \texttt{\small drop} for S-rank SIers ($9.36$) was lower than that for the others (A-rank: $22.54$, B-rank: $16.13$)\footnote{The highest \texttt{\small drop} ratio in A-rank SIers might be related to a large EVS$_\text{end}$. See Section \ref{ssec:evs-results}.}.
These results suggest that the SI$^\text{sent}$ generated by higher-ranked SIers tended to have a higher overall quality.

At the sentence level, S-rank SIers produced the largest number of \emph{bunsetsus} (see \texttt{\small \# bunsetsu per sent.}) among SIers.
A one-way ANOVA detected significant differences among the groups ($F(2, 5818) = 21.88, p < .001$), and the subsequent Tukey's test showed that S- and B-rank SIers produced significantly larger numbers of \emph{bunsetsus} than did A-rank SIers ($p < .001$).
Although the difference between the S- and B-ranks was not significant, the results presented in Section \ref{sec:quality} suggest that SIers with more experience retained more contents in the source sentences and achieved higher quality at the sentence level.

Table \ref{table:corpus-stats} also show that higher-ranked SIers tended to have higher \texttt{\small skip} ratios.
However, the differences among the groups were not statistically significant based on a one-way ANOVA ($F(2, 39) = 0.52, p = .600$).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.4
\subsection{Latency}
\label{sec:latency}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.4.1
\subsubsection{Metrics}
As a latency metric, Ear-Voice Span (EVS) was calculated for each sentence.
Since our data contain information on the start/end times of speech segments, we separately calculated EVS in milliseconds at the beginning and at the end of a sentence\footnote{Owing to the limitations of our data, we calculated a relatively simple EVS as in \citeA{Christoffels-2004}. In other previous studies, time tags were annotated at a finer level of granularity (e.g., every \textit{N} word, every content word), which enables the analysis of SIers' speech rate or the types of words that cause large latency.}.
\begin{align}
{EVS_{start}} & = {start}\ {time_{JA}} - {start}\ {time_{EN}}, \nonumber \\
{EVS_{end}} & = {end}\ {time_{JA}} - {end}\ {time_{EN}}. \nonumber
\end{align}

However, we were unable to calculate the EVS for some sentences because certain segments were divided into multiple segments during sentence-level alignment, and thus start/end times were unavailable.
Furthermore, EVS$_\text{end}$ can become negative if the SIer stops interpreting a sentence in the middle.
These cases were excluded from the analysis.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T8
\begin{table}[b]
\input{02table08.tex}
\caption{Average EVS (seconds) calculated with manually aligned data}
\label{table:result-evs}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.4.2
\subsubsection{EVS Results}
\label{ssec:evs-results}
Table \ref{table:result-evs} compares the latency measured by EVS for the 14 talks based on the manually aligned data.
The A-rank SIers had the largest latency at both beginning and end of sentences, followed by the B- and S-rank SIers.
The amount of latency ranged from 2 to 4 seconds\footnote{Hereafter, EVS results are reported in seconds for convenience of understanding.}, which was in line with previous studies \cite<see>{Robbe-2019}.

However, a relatively large number of EVS took greater values ($>$ 5 seconds).
This motivated us to investigate the relationship between EVS and source sentence length.
A low Pearson's correlation coefficient suggests that the sentence length in the source language did not affect the EVS ($r = .26$ and $.12$ for EVS$_\text{start}$ and EVS$_\text{end}$, respectively), which did not match with the results reported in \citeA{lee-2002}.
The EVS$_\text{start}$ became large because the SIers sometimes did not interpret the earlier part of the sentence, as in this example:
\begin{itemize}
\setlength{\leftskip}{10mm}
    \item[(En)]A week later, Ping was discovered in the apartment alongside the body of her owner, and the vacuum had been running the entire time.
    \item[(A-rank)]そしてずっと掃除機がオンになったまま残されていたんですけれども、[And the vacuum had been running the entire time.] 
\end{itemize}

The EVS$_\text{end}$ results suggest that S- and B-rank SIers tended to wrap up the sentence to a certain extent as the next sentence started, whereas A-rank SIers might have held on the sentence, resulting in a larger EVS$_\text{end}$.
A large EVS$_\text{end}$ appeared to negatively affect the interpreting performance of the subsequent sentence, as reported in \citeA{lee-2002}.
Focusing on the top 10\% of sentences whose EVS$_\text{end}$ value was large ($N = 187$), $56.68\%$ of their subsequent sentences were not interpreted at all (i.e., \texttt{\small drop}) by the A-rank SIers.

The manner in which the EVS and the \texttt{\small drop} ratio vary (wherein the value increases at the A-rank and then decreases at the S-rank) might be an example of U-shaped learning.
U-shaped learning refers to a learning pattern characterized by an initial reduction in performance followed by an improvement.
It occurs when individuals learn complex skills or concepts \cite{Carlucci-2006}.
SIers might attempt to produce a higher-quality SI$^\text{sent}$ as they become more experienced.
However, at the A-rank stage, they might not be sufficiently skilled to control the latency and quality.

Table \ref{table:evs-manual-auto} compares the EVS calculated using the manually and automatically aligned data.
Note that the metric was calculated from SI$^\text{sent}$ in six talks generated by S-rank SIers (see Section \ref{ssec:Sentence-level-Alignment}).
Two-way ANOVA detected significant differences among talks in EVS$_\text{start}$ and EVS$_\text{end}$ ($F(5, 829) = 8.55$, $p < .001$ and $F(5, 761) = 10.80$, $p < .001$, respectively) but not between manually and automatically aligned data ($F(1, 829) = 3.57$, $p = .059$ and $F(1, 761) = 1.68$, $p = .195$, respectively).
No interactions were detected between talks and alignment methods ($F(5, 829) = 0.53$, $p = .751$ and $F(5, 761) = 0.61$, $p = .692$, respectively).
The results of the statistical tests suggest that EVS analyses are possible using automatically aligned data, which permits analyses with very large amounts of data (e.g., NAIST-SIC contains more than 800 such talks).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T9
\begin{table}[t]
\input{02table09.tex}
\caption{Comparison of EVS (seconds) calculated using manually and automatically aligned data}
\label{table:evs-manual-auto}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.5
\subsection{Quality}
\label{sec:quality}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.5.1
\subsubsection{Metrics}
To evaluate the quality of WT$^\text{sent}$ and SI$^\text{sent}$, we used two metrics\footnote{We focused on faithfulness in this paper, although other factors may affect SI$^\text{sent}$ quality (e.g., grammaticality and delivery).}. 

The first was BERTScore \cite{zhang-2020-BERTScore}, which is based on contextualized subword embeddings and is expected to capture meanings rather than surface forms in the manner of BLEU \cite{papineni-etal-2002-bleu}.
It appears to be appropriate for evaluating the particular aspects of SI$^\text{sent}$, where SIers use anticipation, summarization, and generalization.
BERTScore was calculated between SI$^\text{sent}$ (candidates) and WT$^\text{sent}$ (references).

The second metric was the \emph{content-word preservation score} (CWPS), which evaluates how many content words in source sentences are expressed in the WT$^\text{sent}$ or SI$^\text{sent}$.
CWPS was calculated based on the points assigned to phrase-level alignment (see Section \ref{ssec:phrase-level-alignment}).
The CWPS for a given sentence was calculated by adding the points and dividing them by the number of content words in the source sentence.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.5.2
\subsubsection{BERTScore Results}
Table \ref{table:result-bertscore} shows the BERTScore calculated for sentences from the 14 talks.
The precision was higher than the recall for all the three SIer ranks.
The results confirm our intuition because SIers occasionally summarize or generalize the content of the original speech to manage latency, and source sentences are not necessarily interpreted literally.
BERTScore captured the quality of SI$^\text{sent}$ well in the following example:
\begin{itemize}
\setlength{\leftskip}{10mm}
    \item[(En)]We did this experiment for real.
    \item[(WT$^\text{sent}$)]実際にこの実験を行ってみました。
    \item[(A-rank)]これを実際にしました。[Did this for real.] 
\end{itemize}
The F1 score for this example was 0.83.
Although the wording that corresponds with ``did'' is different between the WT$^\text{sent}$ and the SI$^\text{sent}$, BERTScore captured the similarity of the meaning.
However, as shown in the next example, BERTScore did not always work well, especially when SIers used a certain strategy:
\begin{itemize}
\setlength{\leftskip}{10mm}
    \item[(En)]We can all think of some examples, right?
    \item[(WT$^\text{sent}$)]例を挙げる事ができると思います。
    \item[(S-rank)]例えば、[For example.]
\end{itemize}
The F1 score for this example was 0.55.
The SIer selected a simple and short expression to convey the idea of the original utterance, although it was challenging for BERTScore to capture it.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T10
\begin{table}[t]
\input{02table10.tex}
\hangcaption{Comparison of BERTScores among interpreters with different levels of experience. The scores are averages of each sentence, where zero is assigned to \texttt{\small drop} and \texttt{\small skip}.}
\label{table:result-bertscore}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


In a comparison among the three SIer ranks, the S-rank SIers achieved the highest scores for precision, recall, and F1.
A one-way ANOVA detected significant differences among the groups ($F(2, 5045) = 65.80$, $70.10$, and $68.39$ for precision, recall, and F1, respectively; $p < .001$), and the subsequent Tukey's test showed that the differences among all groups were significant ($p < .05$).
The lower scores of the A-rank SIers than those of the B-rank SIers is likely to have been caused by the high \texttt{\small drop} ratio.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.5.3
\subsubsection{CWPS Results}
While the quality of SI$^\text{sent}$ was examined using BERTScore by assuming WT$^\text{sent}$ as the reference\footnote{Interpreting sentences may be more accurate than WT$^\text{sent}$. For example, SI$^\text{sent}$ with a word order similar to that of the source language may better reproduce the information structure.}, CWPS was used to evaluate the quality measurements of WT$^\text{sent}$ and SI$^\text{sent}$ against the source sentences.
Table \ref{table:result-cwps} shows the CWPS calculated for the sentences sampled from the three talks.
The results indicate that WT$^\text{sent}$ achieved the highest scores, followed by S-, A-, and B-rank SIers (except for \texttt{\small Ale}, where the order of A- and B-rank was reversed).
In fact, the low ratios of \texttt{\small drop} and \texttt{\small en null} (8.33 and 0.00, respectively) suggest that the B-rank SIer performed well on \texttt{\small Ale}, which matched the human evaluation results (see Section \ref{sec:human-evaluation}).
One of the human evaluators remarked that key words such as proper nouns were translated effectively or rephrased appropriately to the corresponding Japanese words.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T11
\begin{table}[t]
\input{02table11.tex}
\caption{Comparison of CWPS among WT$^\text{sent}$ and SI$^\text{sent}$ data}
\label{table:result-cwps}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


We also investigated the coverage of content words, where no distinction was made between ``meaning was successfully conveyed'' (earning a point of 1) and ``partially conveyed'' (earning a point of 0.5).
Table \ref{table:cover-ratio-thresholds} shows the number of sentences with different coverage thresholds.
The results suggest that S-rank SIers attempted to interpret more content words than A- and B-rank SIers.
It was noteworthy that the values for the A- and B- rank SIers were almost identical for all the threshold ranges.
Considering the differences in CWPS between A- and B-rank SIers, the results suggest that the SI$^\text{sent}$ generated by A-rank SIers were of higher a quality than those generated by B-rank SIers.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T12
\begin{table}[t]
\input{02table12.tex}
\caption{Comparison of the number of sentences with different cover ratio thresholds}
\label{table:cover-ratio-thresholds}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


These results imply that the higher-ranked SIers generated better interpretations at the sentence level.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.5.4
\subsubsection{Relationship between latency and quality}
Since previous studies have shown that a higher latency damages quality \cite<e.g.,>{lee-2002}, we investigated the relationship between latency and quality based on the EVS$_\text{start}$.
In Section \ref{ssec:evs-results}, the negative effect of a large EVS$_\text{end}$ on the interpreting performance of the subsequent sentence was discussed; in this section, we examine whether a large EVS$_\text{start}$ affects the quality of the sentence being processed.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F3
\begin{figure}[t]
\begin{center}
\includegraphics{31-3ia2f3.pdf}
\end{center}
\hangcaption{Relationship between EVS$_\text{start}$ and BERTScore (F1). $r$ denotes Pearson's correlation coefficient.}
\label{relationship-evs-bertscore}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


The relationship between EVS$_\text{start}$ and the BERTScore of \texttt{\small Ale}, \texttt{\small Nic}, and \texttt{\small Lau} is shown in \mbox{Figure}~\ref{relationship-evs-bertscore}.
The values of Pearson's correlation coefficient were negative.
When the latency increased ($>$ 5 seconds), few SI$^\text{sent}$ had high scores ($> 0.8$).
In addition, when the talk was easy to interpret (\texttt{\small Ale}), the standard deviation of EVS$_\text{start}$ was smaller than that of the other talks (\texttt{\small Ale} $= 1.33$, \texttt{\small Nic} $= 2.25$, and \texttt{\small Lau} $= 2.16$).
Furthermore, the standard deviation of the S-rank SIers' was smaller than that of the other SIers (e.g., S $= 1.06$, A $= 1.68$, and B $= 1.27$ for \texttt{\small Ale}).

These results suggest that a large EVS$_\text{start}$ negatively affected the interpretation quality.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.6
\subsection{Word Order}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.6.1
\subsubsection{Metrics}
\label{ssec:word-order-metrics}
To examine the word order differences between the WT$^\text{sent}$ and SI$^\text{sent}$, we computed Kendall's rank correlation coefficient (tau).
Tau ranges from -1 (if one list is the reverse of the other) to 1 (if the two lists are identical).
Tau was normalized to the range [0, 1]:
\begin{equation}
    normalized \ tau=\frac{tau+1}{2}. \nonumber
\end{equation}
In this study, the metric was calculated based on the chunk-level alignment (see Section \ref{ssec:chunk-level-alignment}).
That is, tau was calculated only from the words appearing in the WT$^\text{sent}$ or SI$^\text{sent}$. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.6.2
\subsubsection{Normalized Tau Results}
Since sentence structures differ between English and Japanese, the larger normalized tau suggests that the SIers adopted a strategy of maintaining the word order of the source language, as in this example:
\begin{itemize}
\setlength{\leftskip}{8mm}
    \item[(En)]If I throw the coin four times / and it's a fair coin, / then the probability / that it comes up four times tails / is 6.25 percent
    \item[(S-rank)]四回投げた時には / \textbf{確率として} / 四回裏が出るのは / 六点二五パーセントです。\\
    {\small [if I throw four times / \textbf{the probability} / that it comes up four times tails / is 6.25 percent]}
\end{itemize}
By first translating ``the probability,'' which is modified by an appositive clause, the SIer retained the English word order.
In WT$^\text{sent}$, on the other hand, the same phrase was translated as ``４回とも裏面が出る / 確率は [it comes up four times tails / the probability],'' which is a typical head-final structure.

The word order differences measured by the normalized tau is shown in Table \ref{table:result-tau-coverage}.
We expected that higher-ranked SIers would have a larger normalized tau, but the value for the B-rank SIers was the largest among the SIers, followed by the S- and A-rank SIers.
This was probably because the content-word coverage ratio was low in the B-rank SI$^\text{sent}$ (Table \ref{table:result-tau-coverage}).
As mentioned previously, we computed the normalized tau only from the words present in the SI$^\text{sent}$.
If a sentence was undertranslated (i.e., a small number of \emph{bunsetsus} in SI$^\text{sent}$), an excessively high value could be assigned. 
An example of this is shown in Table \ref{table:high-tau-example}.
The normalized tau for this sentence was 1.00, but the value might be lower if the phrases ``feel'' or ``have to talk'' were translated.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T13
\begin{table}[b]
\input{02table13.tex}
\caption{Normalized tau and average content word coverage ratio}
\label{table:result-tau-coverage}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T14
\begin{table}[t]
\input{02table14.tex}
\hangcaption{Example of a sentence whose normalized tau is high owing to undertranslation. The numbers preceding Japanese \emph{bunsetsus} represent the order of appearance in the SI$^\text{sent}$.}
\label{table:high-tau-example}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



If the content-word coverage ratio is relevant, the normalized tau for WT$^\text{sent}$ could be excessively low because its content-word coverage ratio was much higher than those of SI$^\text{sent}$.
For a fair comparison, we calculated the average normalized tau only from sentences whose content word coverage was over 0.7 in both S-rank and WT$^\text{sent}$ ($N = 27$), and we observed that the value for S-rank (0.85) was larger than that for WT$^\text{sent}$ (0.78).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.7
\subsection{Human Evaluation}
\label{sec:human-evaluation}
The quality of SI$^\text{sent}$ was further examined through human evaluations.
Three professional translators (i.e., not interpreters) subjectively evaluated the faithfulness of each sentence on a scale of four: 1 (incomprehensible), 2 (poor), 3 (minor errors), and 4 (acceptable).
Table \ref{table:result-human} shows the average sentence scores for each combination of talks and SIer ranks.
Higher-ranked SIers tended to receive higher scores, which matched the BERTScore and CWPS results.
The B-rank SIer interpreted \texttt{\small Ale} effectively, which was mentioned in the translators' overall comments.
The individual differences among SIers (e.g., background knowledge) could have affected the SI$^\text{sent}$ quality, because the same SIers did not necessarily interpret all the three talks. 



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T15
\begin{table}[t]
\input{02table15.tex}
\caption{Comparison of subjective evaluations by three professional translators}
\label{table:result-human}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



Table \ref{table:result-human} also shows that the human evaluation scores were low (generally $< 2$).
One possible reason is the translators' strict adherence to the source sentence structure, as shown in the following example:
\begin{itemize}
\setlength{\leftskip}{10mm}
    \item[(En)]People are motivated by different values perhaps.
    \item[(A-rank)]人のモチベーションは / 違う物によって / 起こってきます。[People's motivation / by different things / is raised.]
    
	(Human evaluation scores: 1, 3, 2) 
\end{itemize}
The verb phrase (are motivated) was interpreted with a noun (motivation) to maintain the English word order, while Rater A indicated the disagreement and assigned one point.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T16
\begin{table}
\input{02table16.tex}
\caption{Correlation between human evaluations and quality metrics}
\label{table:corr-human}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Pearson's correlation coefficient was calculated between the human evaluation scores and the two quality metrics.
CWPS achieved higher correlations with human judgments than BERTScore (Table \ref{table:corr-human}).
However, the correlation coefficient values were not very high.
The relatively small number of sentences (\texttt{\small Ale} $= 29$, \texttt{\small Nic} $= 27$, and \texttt{\small Lau} $= 32$) used in the analysis might have affected the results.
Further research with more data and, more importantly, evaluation scores by SIers is required.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5
\section{Conclusion}
We described the development of NAIST-SIC, which contains SI$^\text{sent}$ generated by SIers with different levels of experience (S-, A-, and B-ranks) from identical source speeches.
Our analyses focused on the latency, quality, and word order, and the results demonstrated that higher-ranked SIers tended to generate a higher quality of SI$^\text{sent}$ and that S-rank SIers controlled the latency and quality better than did the other two ranks.
Certain tendencies observed in the analyses (e.g., large latency for A-rank SIers and low scores on the quality metrics for B-rank SIers; see Sections \ref{sec:latency} and \ref{sec:quality}, respectively) can be considered as aspects that the SIers can improve.
A comparison of the performance of the SIers at different levels can provide useful information for SIer training.

However, the analyses in this paper were based on the SI$^\text{sent}$ generated by two or three SIers for each rank.
It is known that the SIers' personalities or styles affect their translation approaches \cite{mizoguchi-2009}.
The results could be attributed to these rather than the SIers' ranks.
In addition, as discussed in Section \ref{sec:human-evaluation}, some analyses were based on a small subset of data. 
Future studies would involve analyses using larger number of SI$^\text{sent}$ from various SIers. 

The main contribution of this study is the development of a large-scale English-Japanese SIC.
Despite some limitations, this study suggests that differences exist between SI$^\text{sent}$ and WT$^\text{sent}$.
We strongly believe that NAIST-SIC will be a useful resource for the development of automatic SI systems and for the research in interpretation and translation studies.


% Start Acknowledgment
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment
A part of this work was supported by JSPS KAKENHI Grant Numbers JP21H05054 and by JST, the establishment of university fellowships towards the creation of science technology innovation, Grant Number JPMJFS2137.


% Start bibliography
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.7}
    \bibliography{02refs}



% Start biography
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Kosuke Doi}{%
He is a Ph.D. student at the Nara Institute of Science and Technology.
He received his Bachelor of Arts degree from Tokyo University of Foreign Studies in 2010.
He received his Master's degree in linguistics from Tokyo University of Foreign Studies in 2012 and that in engineering from the Nara Institute of Science and Technology in 2022.
He previously worked with Eikoh Inc. from 2012 to 2016 and with Z-kai Inc. from 2016 to 2020.
}

\bioauthor[:]{Katsuhito Sudoh}{%
He is a professor at Nara Women's University.
He received his bachelor's degree in engineering in 2000, and his master's and Ph.D. degrees in informatics in 2002 and 2015, respectively, from Kyoto University.
He worked with the NTT Communication Science Laboratories from 2002 to 2017 and the Nara Institute of Science and Technology from 2017 to 2024.
He works on machine translation and natural language processing.
He is a member of ACL, ISCA, ANLP, IPSJ, ASJ, and JSAI.
}

\bioauthor[:]{Satoshi Nakamura}{%
He is a Professor Emeritus at the Nara Institute of Science and Technology NAIST, a full professor at the Chinese University of Hong Kong, Shenzhen CUHKSZ, and an honorary professor at the Karlsruhe Institute of Technology, Germany.
He received his B.S. degree from the Kyoto Institute of Technology in 1981 and a Ph.D. from the Kyoto University in 1992.
He was the Director of the ATR SLC laboratory from 2000 to 2008 and the Director General of Keihanna Research Laboratories, NICT.
He was a professor and Director of the AHC Laboratory at NAIST from 2011 to 2023.
He is currently a full professor at CUHKSZ.
He was an elected board member of the ISCA in the period of June 2011--2019, an IEEE Signal Processing Magazine Editorial Board member in the period 2012-2015, and an IEEE SPS Speech and Language Technical Committee Member in the period 2013--2015.
He is an ATR Fellow, IPSJ Fellow, IEEE Fellow, and ISCA Fellow.
}
\end{biography}

\biodate   %%%% Following settings will be edited by editors.


\end{document}

