\documentclass[english]{jnlp_3.6}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{hangcaption_jnlp}
%%%%\usepackage{udline}
%%%%\setulminsep{1.2ex}{0.2ex}
%%%%\let\underline=\ul
\allowdisplaybreaks


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
%%%%\usepackage{times}
%%%%\usepackage{url}
%%%%\usepackage{amsmath}
%%%%\usepackage{amsfonts}
%%%%\usepackage{url}
%%%%\usepackage{times}
%%%%\usepackage{url}
%%%%\usepackage{latexsym}
%%%%\usepackage{algorithm,algorithmic}
\usepackage{xcolor}
\usepackage{colortbl,array}
\usepackage{booktabs}
\usepackage{multirow}
%%%%%%\usepackage{subcaption}
%%%%\usepackage[subrefformat=parens]{subcaption}
%%%%\renewcommand\UrlFont{\rmfamily}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{30}
\Number{1}
\Month{March}
\Year{2023}
%%%%\Issuetitle{Special Issue: XXXXXXXX}


\received{2022}{6}{2}
\revised{2022}{9}{22}
\accepted{2022}{10}{13}

\setcounter{page}{30}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Pipeline Signed Japanese Translation Using PBSMT and Transformer in a Low-Resource Setting}
\eauthor{Ken Yano\affiref{UEC} \and Akira Utsumi\affiref{UEC}}
\eabstract{%
We propose a novel pipeline method for translating signed Japanese sentences into written Japanese. Sign languages often suppress functional words such as particles, and most words are not morphologically inflected as they are in spoken languages. Our method explicitly compares and contrasts the two languages and divides the translation process into two tasks: first, it translates glosses into lemmatized Japanese words or phrases, followed by complementing particles and conjugating predicates such as verbs, auxiliary verbs, and adjectives. Our method is especially effective when the size of the parallel corpus is very limited and costly to obtain, but there are plenty of monolingual corpora for the target. Specifically, our method first uses phrase-based statistical machine translation (PBSMT) to map sign glosses to corresponding Japanese words or phrases, and then employs a transformer-based neural machine translation (NMT) model trained with a monolingual corpus to refine the output in the first translation. Experimental results show that our pipeline method outperforms direct PBSMT and competitive NMT models with data augmentation, including back-translation and transfer learning in a low-resource setting with a corpus size on the order of $10^4$ words.
}
\ekeywords{Sign Language Translation, Pipeline Translation, Low-Resource Translation}

\headauthor{Yano and Utsumi}
\headtitle{Pipeline Signed Japanese Translation}

\affilabel{UEC}{}{The University of Electro-Communications}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}
\label{intro}
Sign languages have different vocabularies and grammars from their spoken counterparts.
In Japan, two variants of sign language exist: (1) Japanese Sign Language (JSL), and (2) Manually Coded Japanese (MCJ) \cite{hirohito-2001}.
JSL, which is used by early signers, features a syntax whose word order, language structure, and other characteristics differ from those of spoken Japanese.
Conversely, MCJ is typically used by late signers and people with acquired hearing loss, therefore featuring a much more similar syntax to that of spoken Japanese.
However, researchers found these variants to be used interchangeably without a clear boundary between them.
In this study, we consider an intermediate between JSL and MCJ, called Signed Japanese (SJ), also referred to as Pidgin Sign Japanese by \citeA{hirohito-2001}. The syntax of SJ is similar to that of MCJ in that the word order is equivalent to that of spoken Japanese.

Figure \ref{fig:jsl_sj_mcj} illustrates how the sample sentence ``男がコーヒーを飲んでいる女を見る (A man sees a woman drinking coffee) '' is differently expressed in JSL, MCJ, and SJ.
Each rectangular box is a \emph{gloss} representing the meaning of a delineated sign gesture.
Note that the word order of JSL is different from that of spoken Japanese, SJ, and MCJ.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1
\begin{figure}[b]
\begin{center}
\includegraphics{30-1ia2f1.pdf}
\end{center}
\hangcaption{The same sentence is expressed differently in JSL, MCJ, and SJ. Each rectangular box is a gloss representing the meaning of the delineated sign gesture.  In MCJ, the particles are usually expressed by fingerspelling, often alongside mouthing \protect\cite{hirohito-2001}.}
\label{fig:jsl_sj_mcj}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Translation from sign language to spoken language is typically performed in two steps, as illustrated in Figure \ref{fig:sign_translation}. First, consecutive visual-gestural signs are recognized from a video signal and transformed into a gloss sentence. The gloss sentence is then translated into a spoken sentence.
The present study considers only sign translation, corresponding to the second step in Figure \ref{fig:sign_translation}.
For those interested in the sign recognition process, please refer to relevant works such as those of \citeA{Zuo_2022_CVPR}, \citeA{8598757}, and \citeA{takayama2021}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 2
\begin{figure}[t]
\begin{center}
\includegraphics{30-1ia2f2.pdf}
\end{center}
\hangcaption{Two-step sign language translation. The present study addresses natural language translation, the second step of the pipeline translation given sign language glosses.}
\label{fig:sign_translation}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Current state-of-the-art sign language recognition and translation methods \cite{camgoz2020sign,yin2020better} require large quantities of annotated data and pay little attention to the differences between a sign language and the corresponding spoken language.
Therefore, the success of these approaches heavily relies on large, paired corpora, and studies on resource-poor sign languages, including SJ-Japanese, cannot take advantage of such approaches.

In this study, we focused on the similarities and differences between sign and spoken languages.
In sign languages, function words such as particles and determiners do not tend to be explicitly signed, and inflectional morphemes associated with verbal predicates that express categories, such as tense and mood, are likewise signed infrequently.
For instance, in the case of SJ, post-positional particles `助詞(\emph{jyoshi})' are generally not explicitly signed, and the signs associated with verbal predicates are not conjugated. Conversely, verbs, adjectives and auxiliary verbs need to be conjugated in spoken Japanese according to the context.
Consequently, SJ glosses usually do not include these constructs.

Although gloss notation is commonly used to transcribe sign language resources, its quality and size vary by language \cite{bungeroth2008atis}.
For instance, SJ glosses are heavily polysemous, and their meanings are often context-sensitive. In addition, there are no publicly available parallel corpus encompassing SJ glosses and Japanese sentences. Therefore, we used a small in-house parallel corpus of SJ glosses and Japanese sentences generated in a specific domain.
We provide details pertaining to this parallel corpus and its annotation method in Section \ref{sec:dataset}.

The objective of this study was to solve the problem of translating SJ glosses to Japanese sentences under an extremely low-resource condition, namely when the size of the parallel corpus is on the order of $10^4$ words.
To perform this challenging task, we propose a novel pipeline method that translates SJ gloss sentences into spoken Japanese. To accomplish this, we focused on the linguistic differences between SJ and Japanese. In particular, we attempted to complement missing post-positional particles and estimate the appropriate forms corresponding to the morphological inflections of words for glossed sentences.
Our proposed method first uses phrase-based statistical machine translation (PBSMT) to map SJ glosses onto lemmatized Japanese words.
We then transformed the results into grammatically correct Japanese sentences using transformer-based neural machine translation (NMT) \cite{vaswani2017attention} \cite{sutskever2014sequence}, which was trained using pseudo-parallel corpora generated from a large out-of-domain monolingual corpus.
In this study, we assumed all SJ glosses to be error-free, and did not address how SJ glosses are generated from a gestural signals in the overall framework of sign language translation. However, this assumptions do not limit this study's applicability to the typical two-step sign language translations mentioned previously.

Our experimental results demonstrate that the proposed pipeline method achieves better performance than various baselines that directly translate SJ glosses to Japanese sentences in an end-to-end manner, including PBSMT and various NMT methods under low-resource settings.
We also verified that the proposed pipeline translation method is effective when translating American Sign Language (ASL) glosses to English sentences, and demonstrated it to deliver similar advantages over the baselines in a simulated extremely low-resource scenario.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related work}

\citeA{camgoz2018neural} proposed end-to-end sign language translation in an NMT framework, which enabled the joint learning of the spatial sign representation, underlying language model, and mapping between signs and spoken language using PHOENIX-Weather 2014T \cite{camgoz2018neural}.\footnote{\url{https://www-i6.informatik.rwth-aachen.de/~koller/RWTH-PHOENIX-2014-T/}}
Their later work \cite{camgoz2020sign} further improved the model by introducing a transformer-based architecture that jointly learns sign language recognition and translation while being end-to-end trainable using connectionist temporal classification loss to bind the recognition and translation problems into a single unified architecture.
In a similar line of research, \citeA{yin2020better} proposed the Gloss-to-Text (G2T) model using a transformer-based seq2seq model. They evaluated the performance of PHOENIX-Weather 2014T \cite{camgoz2018neural} and ASLG-PC12 \cite{othman2012English} by varying the number of encoder-decoder layers and embedding schemes.
These existing state-of-the-art end-to-end sign language translation methods rely on large datasets, and are difficult to apply to resource-poor sign language translations, such as SJ to Japanese.

Several attempts have been made to address the resource scarcity of sign language translation.
\citeA{mocialov2018transfer} used transfer learning to model the British Sign Language (BSL) at the gloss level. They first trained a stacked LSTM model or feed-forward model on the Penn Treebank dataset, and then retrained the model either by fine-tuning or layer substitution on the BSL corpus. They claimed that this improved the language modeling of BSL in terms of perplexity.
\citeA{angelova-etal-2022-using} investigated the G2T task in a low-resource setting using two German Sign Language
corpora: PHOENIX-Weather 2014T \cite{camgoz2018neural} and public DGS.\footnote{\url{https://www.sign-lang.uni-hamburg.de/dgs-korpus/index.php/welcome.html}}
They conducted extensive experiments among three groups: two model architectures represented by a recurrent neural network (RNN) and transformer, three tokenization methods for subword, unigram and custom; and two data augmentation methods corresponding to back-translation and paraphrasing via bilingual pivoting.
In contrast, we propose a pipeline system to mitigate the issue caused by resource scarcity.

Low-resource issues also apply to translations between spoken languages.
One limitation of NMT is the requirement for large-scale parallel data for model training.
\citeA{koehn-knowles-2017-six} claimed that NMT underperforms PBSMT when the available parallel training data encompass less than 10 million words (approximately 0.1 million sentences).
However, under low-resource settings without a monolingual corpus, \citeA{sennrich-zhang-2019-revisiting} claimed that the performance of NMT system can equate or exceed that of PBSMT with careful hyperparameter tuning.
Our experiments confirmed \citeA{sennrich-zhang-2019-revisiting}'s claim to fail for our task, as although the NMT output tended to be fluent, it sacrificed faithfulness compared with PBSMT in a low-resource setting.

Numerous algorithms have been proposed to address low-resource NMT.
These methods try to collect auxiliary data to achieve competitive results, and can be categorized according to the data used.
Broadly, three types of auxiliary data may be exploited: monolingual data, data from auxiliary languages, or multi-modal data \cite{Wang2021ASO}.
The first category encompasses approaches such as backward and forward translation, unsupervised NMT, and language model pre-training.
The second category includes multilingual training and transfer learning \cite{nguyen-chiang-2017-transfer,zoph-etal-2016-transfer}.
A more detailed survey of NMT for low-resource domains was provided by \citeA{Wang2021ASO}.
In our experiments, we compared the proposed pipeline method with various baseline NMTs using auxiliary data and transfer learning.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Signed Japanese translation}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Task description}
\label{subsec:task}

Our objective was to translate SJ glosses into Japanese sentences. Because there are no well-known defined descriptive methods for glossing SJ, we adopted a unique approach by representing each gloss as a string formed by a capital `G' followed by an arbitrary string of up to 4 digits.
This encoding scheme effectively unifies polysemous sign gestures to unique single gloss words, thereby reducing the number of class labels required for the classification task at the stage of continuous sign language recognition.

The mapping from gloss symbols starting with `G' to the Japanese words or phrases is defined by a gloss dictionary, which encompasses all the definitions of the 1,295 glosses used in our corpus.
We used a second dictionary to map compound glosses in Japanese. For instance, `図書館(library)' is represented as two glosses: one representing `本(book)', and another representing `建物(building)'. The compound gloss dictionary contains 852 definitions.

Figure \ref{fig:task_desc} illustrates a sample translation from SJ gloss sentences to spoken Japanese text.
On the right, we show a snippet mapping between glosses and  corresponding Japanese words.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 3
\begin{figure}[b]
\begin{center}
\includegraphics{30-1ia2f3.pdf}
\end{center}
\hangcaption{Sample translation from SJ gloss sequence to spoken Japanese text. The table on the right represents a subset of the gloss dictionary used to map each gloss to its corresponding Japanese word or phrase.}
\label{fig:task_desc}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

In gloss notation, the closeness of the two glosses' trailing numbers does not indicate semantic similarity. For instance, there is no semantic similarity between the two glosses `G1' and `G11,' as they correspond to `入籍 (enrollment)' and `証明 (certificate),' respectively, in our corpus. This is dissimilar from words in natural languages, where lexical similarity often assumes semantic relatedness, as in the case of `dog' and `dogs.'
Therefore, each gloss in our notation must be treated as a unique word or token that can not be subdivided.

We also assumed that the vocabulary of the glosses is closed.
This assumption renders the out-of-vocabulary (OOV) issue negligible for our translation task using the gloss dictionary as additional training data.
We imposed this assumption for the following reasons: 1) our corpus is extremely small, with many glosses being singletons; and 2) the vocabulary of glosses is shared with a gestural sign recognition module \cite{takayama2021}, that is, an upper module in the two-step sign translation framework. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Pipeline translation}
\label{subsec:pipeline}

Due to the grammatical differences between sign languages and their spoken counterparts, we argue that a gloss sentence can be more easily mapped to an ungrammatical spoken language sentence by eliminating aspects that are unique to spoken language.
This is particularly true when the size of the parallel corpus is very limited, because doing so makes it effortless to translate a gloss sentence.
In addition, the translation of spoken sentences within both grammatical and ungrammatical languages can be trained using only a spoken language corpus.
We therefore propose a pipeline architecture to translate SJ gloss sentences into spoken Japanese sentences.

We first define terminology for the following discussion.
$\mathcal{G}$ represents a gloss sequence, and $\mathcal{S}$ represents a sequence of Japanese words.
We define two subscripts for $\mathcal{S}$: $pp$, which denotes a \emph{post-positional particle}, and $c$, which denotes \emph{conjugation}, with the prefix $+$ or $-$ for each subscript denoting the presence or absence of post-positional particles and conjugation, respectively.
The definition of these terms are listed in Table \ref{tbl:termdef}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[b]
\input{02table01.tex}
\caption{Definition of terms for the variants of $\mathcal{S}$}
\label{tbl:termdef}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 4
\begin{figure}[b]
\begin{center}
\includegraphics{30-1ia2f4.pdf}
\end{center}
\hangcaption{Sample of pipeline translation from the gloss sequence $\mathcal{G}$ to the target Japanese text $\mathcal{S}_{+pp+c}$. The table on the right shows the POS and lemma of each word of the input.}
\label{fig:pipeline_translation}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The overall pipeline translation system is illustrated in Figure \ref{fig:pipeline_translation}.
In the first stage, the gloss sequence $\mathcal{G}$ is translated to $\mathcal{S}_{-pp-c}$  by mapping each gloss to a Japanese words or phrases.
In the following stage, we attempted to translate $\mathcal{S}_{-pp-c}$ to $\mathcal{S}_{+pp+c}$. We considered three ways to accomplish this task: 1) directly translate $\mathcal{S}_{-pp-c}$ to $\mathcal{S}_{+pp+c}$; 2) first translate  $\mathcal{S}_{-pp-c}$ to $\mathcal{S}_{+pp-c}$,  and then translate $\mathcal{S}_{+pp-c}$ to $\mathcal{S}_{+pp+c}$; and 3) first translate $\mathcal{S}_{-pp-c}$ to $\mathcal{S}_{-pp+c}$, and then translate $\mathcal{S}_{-pp+c}$ to $\mathcal{S}_{+pp+c}$.

We propose the use of PBSMT for the first stage of translation from $\mathcal{G}$ to $\mathcal{S}_{-pp-c}$.
We assume PBSMT to be appropriate for first-stage translation because the bitext data are very limited.
PBSMT is based on the phrase-based translation model \cite{koehn-etal-2003-statistical}, which in turn is based upon noisy channel model. 
Using Bayes rule, the translation is formulated as the probability for translating a foreign sentence $f$ into English $e$ as follows:
\begin{equation}
argmax_{e} p(e|f)= argmax_{e} p(f|e)p(e)
\end{equation}

This allows for a language model $p(e)$ and a separate translation model $p(f|e)$ to collaborate.
Each foreign phrase $f_{i}$ is translated into an English phrase $e_{i}$. The English phrases may be reordered.
Phrase translation is then modeled by a probability distribution $\phi(f_{i}|e_{i})$.

The reordering of the English output phrases is modeled by a relative distortion probability distribution $d(a_{i}-b_{i-1}-1)$, where $a_{i}$ denotes the start position of the foreign phrase that was translated into the $i$th English phrase, and $b_{i-1}$ denotes the end position of the foreign phrase translated into the $(i-1)$th English phrase.
To calibrate the output length, a factor $w$ for each generated English word is introduced in addition to the trigram language model $p_{LM}$.

To summarize, the optimal English output sentence $e_{best}$ given a foreign input sentence $f$ is obtained as,
\begin{align}
e_{best}&=argmax_{e} p(e|f) \\
&=argmax_{e} p(f|e)p_{LM}(e)w^{length(e)}
\end{align}

where $p(f|e)$ is decomposed into
\begin{equation}
p(f^I_1|e^I_1) = \prod_{i=1}^I \phi(f_i|e_i)d(a_i-b_{i-1}-1)
\end{equation}

For the PBSMT of our pipeline system, the foreign and English correspond to $\mathcal{G}$ and $\mathcal{S}_{-pp-c}$, respectively.

For the second-stage translation from $\mathcal{S}_{-pp-c}$ to $\mathcal{S}_{+pp+c}$, we employed transformer-based NMT models that were trained by using a large number of pseudo-parallel corpora generated from the monolingual corpus.
In terms of sequence-to-sequence generation, the particle complement is the insertion, and the conjugation is the replacement operation with respect to the source sequence's tokens.
Accordingly, we considered the following alternative methods to translate $\mathcal{S}_{-pp-c}$ to $\mathcal{S}_{+pp+c}$:

\begin{description}
\item[Trans\_1step] translates $\mathcal{S}_{-pp-c}$ to $\mathcal{S}_{+pp+c}$ by jointly estimating the particle complement and conjunction.
\item[Trans\_2step\_PP\_C] first translates $\mathcal{S}_{-pp-c}$ to $\mathcal{S}_{+pp-c}$ using the particle complement, and then translates $\mathcal{S}_{+pp-c}$ to $\mathcal{S}_{+pp+c}$ using the conjunction.
\item[Trans\_2step\_C\_PP] first translates $\mathcal{S}_{-pp-c}$ to $\mathcal{S}_{-pp+c}$ using the conjunction, and then translates $\mathcal{S}_{-pp+c}$ to $\mathcal{S}_{+pp+c}$ using the particle complement.
\end{description}

Details pertaining to pseudo-parallel corpus generation are provided in Section~\ref{subsec:japanese_corpus}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Dataset}
\label{sec:dataset}

We used two corpora: a small in-house SJ and Japanese parallel corpus, and a large out-of-domain Japanese monolingual corpus. The following subsections describe the two datasets.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{SJ and Japanese parallel corpus}
\label{subsec:parallel_corpus}

A summary of the in-house parallel corpus from SJ glosses to Japanese sentences is presented in Table \ref{tbl:parallel_corpus}. 
The dataset comprises 1,120 sentence pairs with 11,887 glosses from a vocabulary of 1,294, and 12,387 Japanese words from a vocabulary of 2,531.
The average gloss sentence length is 6.9 glosses, with a maximum length of 12 and minimum length of 2, and the average Japanese sentences length is 10.3 words, with a maximum length of 21 and minimum length of 5.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 2
\begin{table}[b]
\input{02table02.tex}
\caption{Parallel corpus from SJ glosses to Japanese sentences}
\label{tbl:parallel_corpus}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The corpus comprises ground-truth gloss transcriptions of signs and their translations into Japanese sentences. Although the sentence form is conversational, all sentences were manually constructed. The corpus is scene-oriented and was generated by simulating conversations at municipal offices, such as requests for a certified copy of the resident register, pension, and unemployment insurance.

As noted in Section \ref{subsec:task}, we developed an original gloss notation method wherein each gloss is represented in the form of $Gn$ with an arbitrary unique number $n$.
The mapping between glosses and their corresponding Japanese words or phrases is provided in a dictionary, which allows glosses to map to multiple Japanese words or phrases.
Because the size of the parallel corpus was very limited, approximately 2.3\% of the glosses were singletons.
To mitigate data scarcity and prevent the OOV issue, we used all dictionary items as additional parallel data for model training. 
Details are described in Section \ref{sec:experiments}.

Our corpus currently includes only scene-orientated sentences, and the glosses used are limited in size and coverage.
In fact, approximately 8 to 10K gloss vocabularies have been estimated for JS. 
Consequently, our corpus represents less than 15\% of the total glosses in JSL or SJ.
%
Apparently, the low gloss coverage of our corpus may be an issue outside our domain scenario; however, this issue can be circumvented by collecting data from a broad range of domains in the future.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Out-of-domain Japanese monolingual corpus}
\label{subsec:japanese_corpus}

We used a subset of the Balanced Corpus of Contemporary Written Japanese (BCCWJ)\footnote{\url{https://pj.ninjal.ac.jp/corpus_center/bccwj/en/}} as the monolingual Japanese corpus.
Although we also considered using the Corpus of Spontaneous Japanese (CSJ),\footnote{\url{https://clrd.ninjal.ac.jp/csj/en/}} the sentences therein were primarily transcribed from voice data, and their forms are relatively informal compared with those in the parallel corpus. Therefore, we opted to use the BCCWJ in our task.

Because the corpus encompasses various Japanese sentences from multiple domains, we selected a subset by matching data endings in patterns such as 〜か \emph{ka} (question), 〜しました \emph{shimashita} (admit), 〜ほしいです \emph{hoshiidesu} (ask), 〜います \emph{imasu} (confirm), 〜します \emph{shimasu} (intend), and 〜希望する \emph{kibousuru} (desire).
We selected such patterns that the selected sentences were similar to the target Japanese in the parallel corpus.

As shown in Table \ref{tbl:monolingual_corpus}, the selected subset encompasses 188,828 sentences with 3,676,936 Japanese words from a vocabulary of 68,005 words, covering 89.5\% of the Japanese vocabulary contained in our parallel corpus, as shown in Table \ref{tbl:parallel_corpus}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[t]
\input{02table03.tex}
\caption{Out-of-domain Japanese monolingual corpus}
\label{tbl:monolingual_corpus}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

To train our pipeline model, we used this corpus to create pseudo-parallel corpora.
First, we designed a function $f_{conv}$ that takes as input a Japanese sentence and two parameters $rp$ and $lemm$, which specify whether to remove particles from and lemmatize words in the input, respectively. 
The function returns the ungrammatical Japanese sentence as the output.
Using the function $f_{conv}$, we can generate any pair of pseudo-parallel corpora between the two variants of $\mathcal{S}$ in Table \ref{tbl:termdef} from the Japanese Monolingual Corpora. 
We also employed this function to generate a parallel corpus between $\mathcal{G}$ and $\mathcal{S}_{-pp-c}$.
Here, we assume that $\mathcal{S}_{-pp-c}$ and $\mathcal{S}_{-pp-c}$, generated from the monolingual and parallel corpora, respectively, grammatically compatible, including word order, as our translation system targets SJ.

To implement the noising function $f_{conv}$, we unconditionally lemmatized all verbs, auxiliary verbs, and adjectives and removed all the particles except the final particle `か(ka)'.
We followed a rule-based approach to implement  $f_{conv}$ to conform to the dissimilarity between SJ glossed sentences and spoken Japanese sentences.
However, we note that the implementation of $f_{conv}$ should conform to the gloss definitions of the sign language under study and be customized accordingly.

For tokenization, POS tagging, and lemmatization of Japanese sentences, we used MeCab with IPA Dictionary.\footnote{\url{https://pypi.org/project/mecab-python3/}}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experiments}
\label{sec:experiments}

Because our parallel data were very limited in size, we applied 10-fold cross-validation to evaluate our models.
For the setup, we randomly split the parallel corpus into 10 folds. For each experiment, we used eight folds for training and two folds for validation and testing. 
We conducted ten iterations of the experiments by alternating the folds used for training, validation, and testing.
The average sizes of the training, development, and testing sets were 896, 112, and 112 sentences, respectively.

When splitting the parallel corpus, we set aside the additional parallel data from the gloss dictionary, and appended them to the resulting training data for each experiment.
Therefore, the last component of each fold's training data contained the same parallel data from the gloss dictionary.
After adding the dictionary items, the volume of training data increased to approximately 3.6K.

We measured the performance by averaging the results of 10-fold experiments.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Pipeline Model}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection*{PBSMT}

We used Moses \cite{koehn-etal-2007-moses} to implement PBSMT. 
To train PBSMT, we used the pseudo-parallel corpus between $\mathcal{G}$ and  $\mathcal{S}_{-pp-c}$ generated from the parallel corpus.
We used a modified 3-gram Kneser-Ney to implement the language model, and a monolingual corpus in conjunction with a parallel corpus to train it.
GIZA++ \cite{och-ney-2000-improved} was used to train the phrase-translation table.
The trained model contained 1,295 and 2,500 vocabularies for $\mathcal{G}$ and $\mathcal{S}_{-pp-c}$, respectively.
 
During decoding, Moses scored the translation hypothesis using a log-linear model. Several tuning algorithms have been introduced to optimize the linear model's weights \cite{neubig-watanabe-2016-optimization}. 
In the experiment, we compared the performance with no tuning, MERT \cite{och-2003-minimum} tuning, and PRO tuning \cite{hopkins-may-2011-tuning}, with the best tuning configuration to connect with the later module.

For the decoding, we set the hypothesis stack to 100 and the phrase distortion-limit to 6. All other parameters were set to the default values of Moses.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection*{Transformers}

To train our pipeline model's transformers (Trans\_1step, Trans\_2step\_PP\_C and \linebreak
Trans\_2step\_C\_PP), we used fairseq \cite{ott2019fairseq} with the default network parameter as shown in Table \ref{tbl:pipeline_transformer}.
This parameter yielded superior performance compared to that of baselines in a preliminary experiment.
We used BPE \cite{sennrich2016neural} with a merge count of 10,000 for subword tokenization, which produced approximately 16K subwords in both the source and target.
For comparative purposes, we also tested small-capacity transformers by using the limited word vocabularies that exist in the parallel corpus.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 4
\begin{table}[t]
\input{02table04.tex}
\caption{Pipeline's transformer architecture for Trans\_1step, Trans\_2step\_PP\_C and Trans\_2step\_C\_PP}
\label{tbl:pipeline_transformer}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We used the Adam optimizer \cite{KingmaB14} with an initial learning rate of $\text{5e}-4$, and the inverse-sqrt scheduler with a warmup update of 4,000.
For loss, we employed label-smoothed cross-entropy with a smoothing rate of 0.1.
We set the mini-batch size to 4,096 maximum tokens without using the shared decoder input-output embedding.

For each transformer, we first pre-trained the base model from the pseudo-parallel corpus generated from the monolingual corpus by allocating data among training, development, and testing at an 8:1:1 ratio, and continued training with early stopping.

After pretraining, all base models were fine-tuned using a parallel corpus with early stopping.
At the testing time, we used the PBSMT output as the input, as well as a beam search of width 5 for decoding.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Baseline Models}

For the baselines, we examined PBSMT and several transformer-based NMT models to translate $\mathcal{G}$ to $\mathcal{S}_{+pp+c}$ directly, which we refer to as the G2T models in the forthcoming discussion.
Each G2T baseline is described in the following subsections.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{G2T\_PBSMT}

G2T\_PBSMT is equivalent to the PBSMT used in our pipeline system, except that it directly translates $\mathcal{G}$ to $\mathcal{S}_{+pp+c}$.
The baseline likewise employs the same configuration as the PBSMT of the pipeline system, along with the training and decoding methods.
The only difference is the size of target vocabulary, which changed from 2500 to 2600 by including functional words. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{G2T\_Transformer (G2T\_Trans) }

G2T\_Trans is a transformer-based NMT.
Because the model can only use a small subset of the parallel corpus, appropriate hyperparameters tuning becomes crucial \cite{sennrich-zhang-2019-revisiting}.

Table \ref{tbl:hyper-parameter_G2T} lists the range of randomly selected hyper-parameters, along with the optimal values found. 
For this baseline, we applied word-based tokenization, which yielded 1300 and 2630 vocabularies for the source and target, respectively.
We employed the same optimizer and training scheduler as for the transformers of the pipeline system.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 5
\begin{table}[b]
\input{02table05.tex}
\caption{Hyper-parameter search results of G2T\_Transformer}
\label{tbl:hyper-parameter_G2T}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{G2T\_SLT}

G2T\_SLT was first proposed by \citeA{yin2020better}. As it is based on a transformer, it is essentially equivalent to G2T\_Trans; however, it was implemented using the OpenNMT library.\footnote{\url{https://opennmt.net/}} All codes for training and decoding processes were provided from the authors,\footnote{\url{https://github.com/kayoyin/transformer-slt}} and the model configuration was the same as that of G2T\_Trans.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{G2T\_Transformer with Data Augmentation (G2T\_Trans\_DA)}

For G2T\_Trans\_DA, we have tested several existing data augmentation techniques. 
\pagebreak
The first method, which uses a monolingual corpus as additional training data, was proposed by \cite{sennrich-etal-2016-improving}.
Accordingly, we randomly selected a subset of the monolingual corpus as additional parallel training data, with the corresponding source code set to NULL.
We adjusted the quantity of augmented data between 2,000, 4,000, 8,000, and 16,000, respectively.

The second method uses back-translation \cite{sennrich-etal-2016-improving} to generate additional parallel training data.
To accomplish this, we first trained the back-translation model using the paired corpus to translate $\mathcal{S}_{+pp+c}$ to $\mathcal{G}$. The back-translation model employed the same model architecture as G2T\_Trans, except the source and target were exchanged.
After training the model, we generated back-translated data from the monolingual corpus.

Because the back-translated data contains many unknown words (UNKs), appropriate filtering procedures were applied, with all remaining words used as candidates for additional training data.
4,939 additional parallel data were used in total.
To verity the effect of augmented data, we experimented with three settings to extend the original training set: 1) using all the back-translated data; 2) using the best 2,000 back-translated data in terms of the average output token probability from the back-translation model; and 3) using all the back-translated data by upsampling the original training dataset to match the volume of back-translated data.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{G2T\_Transformer with Transfer Learning (G2T\_Trans\_TL)}

For G2T\_Trans\_TL, we employed transfer learning \cite{kocmi-bojar-2018-trivial,zoph-etal-2016-transfer,nguyen-chiang-2017-transfer} by adopting knowledge learned from relevant translation tasks.
To this end, we first trained two parent transformer-based NMTs to translate 1) from Chinese to Japanese and 2) from English to Japanese using the Asian Scientific Paper Excerpt Corpus (ASPEC) \cite{NAKAZAWA16.621}.
All original training, validation, and testing dataset were provided by ASPEC.

Because the ASPEC dataset is relatively large, we configured the model architecture to be equivalent to that of the pipeline system transformer, as shown in Table \ref{tbl:pipeline_transformer}, with the exception of tokenization.
We tested two tokenization methods by altering the BPE code size between 10,000 and 20,000 to verify the effect of the vocabulary size on the results of the child model's output.
We used MosesTokenizer\footnote{\url{https://github.com/luismsgomes/mosestokenizer}} and jieba\footnote{\url{https://github.com/fxsjy/jieba}} to tokenize the English and Chinese sentences, respectively.

To share the source vocabulary between the parent and child models, as well as prevent the gloss words of the child models from being subdivided, we appended all the gloss words multiple times to the parallel training data during training BPE subwords.

The optimizer and training scheduler were the same as those for the transformers of the pipeline system.
The two pre-trained parent models were fine-tuned using a parallel corpus to translate $\mathcal{G}$ to $\mathcal{S}_{+pp+c}$.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 6
\begin{table}[b]
\input{02table06.tex}
\hangcaption{Performance comparison between G2T baselines and proposed pipeline models, with the best scores for each metric denoted in bold. All baselines with the best model configuration according to preliminary experiments are marked with $\star$. The results of these preliminary experiments are provided in Appendix \ref{appendix:baseline}. {\dag} means that the source gloss symbols were effectively processed as words.
In the pipeline, transformers marked with $\ddag$ use the same configuration as G2T\_Trans except for the vocabularies. 
The \emph{corpus} column specifies the kinds of corpora used to train the models: P (parallel corpus in Section~\protect\ref{subsec:parallel_corpus}), M (monolingual corpus in Section~\protect\ref{subsec:japanese_corpus}), and A (ASPEC). For PBSMT, M is used solely for language modeling. Any corpora used for the fine-tuning phase are underlined. * denotes a statistical significance ($p < 0.05$) compared with G2T\_PBSMT.}
\label{tbl:performance}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Results}

Table \ref{tbl:performance} lists the comparative results of 10-fold experiments 
\pagebreak
for the baselines and our pipeline model.
For G2T\_PBSMT, G2T\_Trans\_DA, and G2T\_Trans\_TL, 
we recorded only the best results from the tuning experiments. Details pertaining to those experiments are listed in Tables \ref{tbl:g2t-pbsmt}, \ref{tbl:g2t-trans-da}, and \ref{tbl:g2t-trans-tl} in Appendix \ref{appendix:baseline}.
We used BLEU \cite{papineni2002bleu} and Unigram chrF \cite{popovic-2015-chrf} to evaluate model performances.
To calculate chrF, we first split the reference and hypothesis sentences into particles, verbs, and other POSs, and subsequently scored the unigram chrF independently for each filtered word sequence. 
We employed these metrics to compare the results of G2T baselines with those of the pipeline model, which selectively complements particles, conjugates verbal predicates, and copies other words from PBSMT output.
Along with these scores, the \emph{tokenization} column indicates the type of tokenziation --- word or BPE --- used by the model used, and the \emph{\#vocab} column indicates the size of the corresponding source and target vocabularies.

We enclosed M for PBSMT in parenthesis because it is only used to train the language model and does not increase the target vocabulary compared with G2T\_Trans.


We confirmed that G2T\_PBSMT outperformed all other transformer-based baselines by a large margin, demonstrating the superiority of PBSMT over NMT in a low-resource setting, corresponding with results obtained in a previous study \cite{koehn-knowles-2017-six}.

Although \citeA{sennrich-zhang-2019-revisiting} claimed that NMT can perform on par with PBSMT in a low-resource setting by careful hyper-parameter tuning, the smallest corpus size they experimented with was on the order of $10^5$, one order of magnitude above our corpus size of $10^4$.
Our results verify that even with carefully tuned hyperparameter, G2T\_Trans underperforms compared to G2T\_PBSMT.

Focusing on the unigram chrF results, we observed comparable performance for all the transformer-based baselines, and noticed that G2T\_PBSMT's chrF for Verbs and Others was much higher than those of the transformer-based G2T models.
This indicates that the results of G2T\_PBSMT were lexically more faithful to the references than those of the NMT-based models; accordingly, more rewriting occurred in the results of the NMT-based baselines.
We hypothesize that in an extremely low-resource setting, transformer-based baselines fail to learn the correct lexical mapping between glosses and corresponding Japanese words, despite including a gloss dictionary within the training data.

The bottom section of Table \ref{tbl:performance} lists the results of the pipeline models.
The upper section shows the results obtained by PBSMT --- that is, the first part of the pipeline that translates $\mathcal{G}$ to $\mathcal{S}_{-pp-c}$ --- whereas the lower section shows results of the transformer-based NMT --- that is, the second part of the pipeline that translates $\mathcal{S}_{-pp-c}$ to $\mathcal{S}_{+pp+c}$.

Compared to the results obtained by G2T\_PBSMT, the intermediate results obtained by PBSMT indicate substantially higher performance, although the references and task were clearly different. 
We placed parentheses around the scores because they were evaluated on the pseudo-target specified by $\mathcal{S}_{-pp-c}$.
This is because PBSMT in the pipeline does not need to estimate the post-positional particles and conjugate forms of words, which simplifies the task compared to that of G2T\_PBSMT.
The highest BLEU was obtained when the weights were tuned using PRO, similar to the results obtained for G2T\_PBSMT.

The six final rows show the performance of transformer-based models with three alternative architectures (see Section \ref{subsec:pipeline}), with the input taken from the output of PBSMT (PRO).
To analyze the impact of the vocabulary size and the model capacity on performance, we also conducted evaluations using the same transformer as that used for G2T\_Trans, except for the vocabularies, which were prepared for the translation task from $\mathcal{S}_{-pp-c}$ to $\mathcal{S}_{+pp+c}$ including only words existing in the parallel corpus.
Models 4\textsuperscript{$\prime$}, 5\textsuperscript{$\prime$}, and 6\textsuperscript{$\prime$} represent comparative transformers with a small vocabulary, corresponding to the original transformers with a large vocabulary indexed by the numbers 4, 5, and 6, respectively.

We verified that among these models, model 4, which jointly estimates the complement of post-positional particles and the conjugation of words, achieved the best BLEU.
Compared with G2T\_PBSMT --- that is, the best G2T baseline --- BLEU improved by 1.78 points.
When comparing Trans\_2step\_PP\_C and Trans\_2step\_C\_PP, the latter exhibited higher performance, which suggests that the conjugation of words is not significantly dependent on but may help infer the complement of particles.

In addition, we acknowledged that the small-vocabulary transformers performed comparably with the original transformer, with only a slight decrease in performance except for \linebreak
Trans\_2step\_C\_PP$\ddag$.
This indicates that the superior performance of the pipeline models over the baselines is not simply a result of the increased vocabulary and transformer capacity.

We claim that the original transformers with a large subword vocabulary are more robust and versatile because they enable us to reuse the models even if the parallel corpus is changed without causing UNK issues for unknown words.
However, a transformer with a limited word vocabulary needs to be trained from scratch whenever the parallel corpus is changed. 
Therefore, the following discussion only considers the original transformers with a large subword vocabulary.

By observing the results of unigram chrF, we confirmed that the scores of Others did not change significantly between the first and second parts of the pipeline, which means that \linebreak
transformer-based NMTs managed to copy words that do not require any modifications. 
The scores of Verbs were higher than those of Particles, which indicates that the estimation of conjugational forms was more accurate than that of the complement of particles.
Note that few post-positional particles, such as the final particle `か(\textit{ka}),' existed in $\mathcal{S}_{-pp-c}$; hence, the chrF of particles for PBSMT was not zero.

The non-parametric permutation test indicated a statistical significance ($p < 0.05$) among the BLEU scores between the pipeline of PBSMT(PRO)+Trans\_1step and the baseline of \linebreak
G2T\_PBSMT.
We also confirmed that the pipeline model performed better than all the other baselines, with a statistical significance of ($p < 0.01$).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 7
\begin{table}[b]
\input{02table07.tex}
\hangcaption{Error propagation analysis of our pipeline model. GS indicates gold standard performance; that is, the best possible performance assuming the input contains no errors. By contrast, EP indicates error propagation performance, wherein the input contained errors propagated from the previous step.}
\label{tbl:error_propagation}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Analysis}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Error propagation of the pipeline model}

Table \ref{tbl:error_propagation} presents the results of an error propagation analysis for the pipeline method by grouping the models with respect to their translation tasks. In the table, GS indicates the gold standard performance --- that is, the best possible performance assuming that the input contains no errors.
In contrast, EP indicates the error propagation performance, wherein the input contains errors propagated from the previous step.

We compared the GS BLEUs of the complement of particles and conjugation of words by transformers, and found that the latter achieved a much higher BLEU by more than 15 points, which suggests that the conjugation of words is much simpler than the complement of particles.
Remarkably, the EP BLEU scores were significantly lower than the corresponding GS BLEU scores, by approximately 15 to 20 points, on average.
We can also observed that these errors primarily originated from the PBSMT results.

By observing that the EP chrF of Others for all models was mostly constant at approximately 85, we can verified that transformers can copy phrases from the PBSMT results when they require no editing.

In terms of the complement of particles, the EP chrF improved from 71.9 to 73.5 between $\mathcal{S}_{-pp-c}$ to $\mathcal{S}_{+pp-c}$ and $\mathcal{S}_{-pp+c}$ to $\mathcal{S}_{+pp+c}$, respectively.
This suggests that the conjugation of words somehow helped complement the particles.

Conversely, for the conjugation of words, we observed that the EP chrF of Verbs degraded from 84.3 to 83.6 between $\mathcal{S}_{-pp-c}$ to $\mathcal{S}_{-pp+c}$ and $\mathcal{S}_{+pp-c}$ to $\mathcal{S}_{+pp+c}$, respectively.
The results of the complement of particles appear to adversely affect the estimation of the conjugation of words;
that is, the conjugation of words should be applied prior to the complement of particles.

Despite this, the EP BLEU of Trans\_1step --- that is, joint estimation of the complement of particles and the conjugation of words --- achieved the best BLEU at 57.43, compared with the Trans\_2step methods.

From this analysis, we hypothesize that the aforementioned joint estimation is more advantageous because it effectively marginalizes the word probability distribution from the two tasks, thereby delivering higher accuracy than the independent estimation of the two tasks in a cascade.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 5
\begin{figure}[p]
\begin{center}
\includegraphics{30-1ia2f5.pdf}
\end{center}
%%%%\label{label:parallelCorpus}
%%%%\label{label:monoCorpus}
\hangcaption{Sensitivity analysis to the change in parallel and monolingual corpus size for the pipeline of PBSMT and NMT\_1step. In the top plot (a), we decreased the relative size of the parallel corpus from 1.0 to 0.5 by 0.1 intervals, whereas in the bottom plot (b), we decreased the size of the monolingual corpus by a factor of 2 at each step. The scores reflect the average of 10-fold cross validation with std. errors.}
\label{label:corpusSize}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Performance sensitivity to corpus size}

We analyzed how the corpus size affects the performance of the pipeline model with PBSMT and Trans\_1step. 
First, we decreased the parallel corpus size from 100\% to 50\%, with a 10\% decrease in each iteration. 
%%%%Figure \ref{label:parallelCorpus} shows the results of PBSMT 
    Figure \ref{label:corpusSize}a shows the results of PBSMT 
in the pipeline ($\mathcal{G}$ to $\mathcal{S}_{-pp-c}$) and direct G2T\_PBSMT ($\mathcal{G}$ to $\mathcal{S}_{+pp+c}$).
The BLEU scores gradually decreased as the corpus was downsized, and there was a constant gap between the BLEU scores of the pipeline and those of the G2T modes.    
From these results, we can speculate that performance has not yet reached a plateau, which clearly indicates the scarcity of the parallel corpus.

%%%%Figure \ref{label:monoCorpus} shows the change 
    Figure \ref{label:corpusSize}b shows the change 
in BLEU scores of the Trans\_1step as the size of the monolingual corpus was halved over eight consecutive steps, coupled with PBSMT trained with a whole parallel corpus.      
The red plot was obtained when we propagated errors from the PBSMT, whereas the blue plot was obtained when we used the gold standard $\mathcal{S}_{-pp-c}$. 
We can verify the performances mostly plateaued when the size of the monolingual corpus exceeded over 3M. 
This suggests that using solely the monolingual corpus, the model is able to successfully learn the tasks of complementing particles, conjugating words, and copying the remaining words.
The performance gap between the two modes primarily results from the PBSMT output. 
These results clearly indicate that the scarcity of the parallel corpus is a bottleneck for overall performance.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Qualitative analysis}

Table \ref{tbl:qualitative_analysis} shows five sample translation results obtained by the two optimal baselines --- G2T\_PBSMT and G2T\_Trans\_TL$\star$ --- as well as the proposed PBSMT+Trans\_1step pipeline. Error phrases are denoted in red.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 8
\begin{table}[b]
\input{02table08.tex}
\hangcaption{Qualitative analysis of G2T\_PBSMT and G2T\_Trans\_TL$\star$, and the proposed PBSMT+Trans\_1step pipeline. Words or phrases in red are erroneous.}
\label{tbl:qualitative_analysis}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We confirmed that although the results of G2T\_Trans\_TL$\star$ represent the most fluent Japanese speech, they occasionally lack faithfulness, as in the examples of A and E, whose semantic meanings are far from the reference.
By contrast, the results of G2T\_PBSMT are mostly correct in content, although the fluency is much worse than obtained by G2T\_Trans\_TL$\star$.

We conclude that the results of the proposed PBSMT+Trans\_1step pipeline are the most admissible to the reference regarding faithfulness and fluency.
However, there is significant room for improvement in the quality of pipeline output. In future work, we may need to seamlessly integrate the outputs of PBSMT and the transformer by exploiting the faithfulness of the former and fluency of the latter.

Moreover, our parallel corpus did not include any non-manual signals (NMSs), such as facial expressions, eye gaze, mouthing, or movement of the head and shoulders.
However, NMSs act as grammatical markings for syntactic information \cite{valli2011linguistics,koizumi2002annotated}.
For instance, the modality of verbs, such as the obligation in the examples of B and C, is often co-articulated with NMSs.
Accordingly, another line of future work may address a streamlined annotation method to reflect not only glosses but also NMS signatures.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}

We proposed a pipeline method to translate SJ glosses into spoken Japanese in a very low-resource setting, where the size of the parallel corpus is on the order of $10^4$ words. 

Specifically, our method focuses on grammatical differences between SJ glosses and Japanese sentences, particularly the omission of post-positional particles and the lemmatization of words in the former.
Our pipeline model consists of a PBSMT and transformer-based NMT.
First, the PBSMT maps sign glosses to lemmatized Japanese words or phrases. The transformer-based NMT, which is trained with a pseudo parallel corpus generated from a large monolingual corpus, then transforms the initial ungrammatical Japanese into a grammatical form by complementing post-positional particles and conjugating verbs, auxiliary verbs, and adjectives.

We compared our pipeline model with various direct translation methods using PBSMT and transformers with existing techniques, such as data augmentation and transfer learning, for low-resource settings.
We also found that jointly estimating the complement of particles and conjugation of words yields higher performance than separately executing the two inference tasks in cascade.

We also experimented with the pipeline method for the translation from ASL glosses to English. The results in Appendix \ref{appendix:asl} exhibit a similar advantage of our pipeline method against the baseline NMT methods, demonstrating that our approach is language-agnostic. 

In the experiment, we assumed the sign language gloss vocabulary to be closed and utilize a dictionary mapping SJ glosses to Japanese words and phrases as an additional parallel training data.            
We believe that this assumption is plausible for end-to-end sign translation from a visually signed language to a spoken language.
In this system, sign glosses are given from a gestural sign recognizer; hence, the vocabulary of glosses needs to be shared with the sign language translation model in advance.

There are several directions for future research in this area.
First, we used the pattern recognition of the lexical endings to select sentences in the BCCWJ for the monolingual corpus. 
However, this method does not necessarily select sentences with similar contexts or topics in the parallel corpus.
To address this issue, we can train a language model using a parallel corpus, and then use it to evaluate the selected sentences with perplexity \cite{moore-lewis-2010-intelligent}.
The generated monolingual corpus is expected to be more mutually coherent with the parallel corpus.

Furthermore, this work assumes that there are no differences in word order between sign language and spoken Japanese, as we only targeted SJ; however, such differences often exist, as in the case for JSL and spoken Japanese. 
Moreover, signers often intermingle JSL, SJ, and MCJ in conversations. 
Hence, future work should address the issue of code-switching and the word order of the sign language translation.

Finally, to improve performance, we are interested in exploring existing methods, such as NMT with copy mechanism \cite{gu-etal-2016-incorporating} to ensure efficient transferring between gloss sentence and target sentence, and non-autoregressive methods \cite{gu-etal-2017-non-auto} to robustly post-edit the output of the translation.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment

This work was supported in part by the joint project ``Research and development on a mutual communication system that connects hearing-impaired and hearing people'' between the University of Electro-Communications and SoftBank Corp.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.7}
\bibliography{02refs}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\appendix

\vspace{-1\Cvs}
\section{Model tuning of G2T baselines}
\label{appendix:baseline}


Table \ref{tbl:g2t-pbsmt} shows the results of weights tuning for G2T\_PBSMT. The best BLEU score (55.67) was achieved when using PRO, with a 1.3 increase compared to the case without tuning.

Table \ref{tbl:g2t-trans-da} lists the results obtained by G2T\_Trans\_DA by demonstrating the effect of data-augmentation on top of G2T\_Trans. We repurposed the source and target dictionaries generated from G2T\_Trans in this experiments, as an increased target vocabulary size did not lead to higher performance in a preliminary experiment.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 9
\begin{table}[b]
\input{02table09.tex}
\caption{Performance of G2T\_PBSMT}
\label{tbl:g2t-pbsmt}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 10
\begin{table}[b]
\input{02table10.tex}
\caption{Performance of G2T\_Trans\_DA}
\label{tbl:g2t-trans-da}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We acknowledge that data augmentation has a negative effect on performance in certain cases, especially when the amount of augmented data exceeds the original training dataset size.
The best BLEU score (48.22) was achieved when all back-translated data were used with upsampling of the original training data to match its size with the augmented data.

Lastly, Table \ref{tbl:g2t-trans-tl} lists the results obtained by G2T\_Trans\_TL, demonstrating the effect of transfer learning. These results indicate that the parent model `zh-ja' is more favorable than the parent model `en-ja'.
Although the reasons behind this were not fully analyzed,  the parent model's performance may positively impact the child model.
To verify the impact of vocabulary size, we alternated BPE code sizes between 10,000 and 20,000. The results show that a larger vocabulary size leads to higher performance in both cases.
Overall, the best BLEU score (52.07) was achieved when the `zh-ja' parent model was used with a BPE code size of 20,000.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 11
\begin{table}[t]
\input{02table11.tex}
\caption{Performance of G2T\_Trans\_TL}
\label{tbl:g2t-trans-tl}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{A case study with American Sign Language (ASL)}
\label{appendix:asl}

To confirm that the proposed pipeline approach works for other sign language translations, we conducted a case study using an English-ASL gloss parallel corpus \cite{othman2012English}.
The corpus was generated using a rule-based method that transformed English sentences to ASL glosses by analyzing parts-of-speech within the former.
It comprises more than 100 million pairs of sentences between English and ASL glosses.
For this experiment, we considered one subset of the paired corpus,\footnote{\url{https://achrafothman.net/site/english-asl-gloss-parallel-corpus-2012-aslg-pc12/}} and randomly sampled 1,200 English-ALS glosses to simulate an extremely low-resource setting as our corpus.
The following example represents two sample sentences from the corpus \cite{othman2012English} for translation between ASL and English.

\begin{itemize}
\item Gloss: HOW DO X-WE MAINSTREAM QUESTION AND PROBLEM ROMA IN SE INSTRUMENT AND POLICY ?
\item EN: How do we mainstream the question and the problems of the Roma in these instruments and policies?
\item Gloss: NL CAR INDUSTRY BE DESC-VERY DESC-IMPORTANT SECTOR FOR EMPLOYMENT IN BELGIUM .
\item EN: NL the car industry is a very important sector for employment in Belgium.
\end{itemize}

It is apparent that a clear one-to-one matching exists between glosses and lemmatized content words without the reordering of words.
We also note that the determiners and prepositions `of' are not included in the gloss. Additionally, some glosses, such as those corresponding to pronouns, have specific forms that start with, for example, `X-.'

We considered the Europarl (German-English, 189~MB) \cite{koehn-2005-europarl} corpus, and extracted the target side as the monolingual corpus.
To generate a corresponding pseudo-parallel corpus, we designed a function $f_{conv}$ that takes an original English text and converts it by removing the determiners and instances of preposition `of,' as well as lemmatizing words.
We used spaCy for the morphological analysis of the English input.
Furthermore, we generated a gloss dictionary by aligning the ASL glosses and corresponding corrupted English sentences of the corpus using fast-alignment \cite{dyer-etal-2013-simple}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 12
\begin{table}[b]
\input{02table12.tex}
\hangcaption{Performance of the ASL-English G2T\_PBSMT and G2T\_Trans\_TL baselines, as well as the proposed PBSMT+Trans\_1step pipeline method. The best scores are denoted in bold. $\mathcal{S}^\dagger$ indicates the analogy of the definition in Table \ref{tbl:termdef} with regard to ASL-English translation.}
\label{tbl:asl-en}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

As in the experiments translating SJ glosses to Japanese words, we applied 10-fold cross-validation.
We examined G2T\_PBSMT and G2T\_Trans\_TL as representative G2T baselines.
For G2T\_Trans\_TL, we used Europarl (German-English) \cite{koehn-2005-europarl} to train the parent model.
Note that to ensure the ASL glosses were not subdivided, we applied the same technique used for SJ glosses, wherein multiple copies of gloss definitions were appended to the BPE training data. 
Hence, in principle, every ASL gloss is treated as an atomic word to be consistent with our encoding method.

Table \ref{tbl:asl-en} presents the experimental results.
For G2T\_PBSMT, we obtained the best BLEU when neither MERT nor PRO weight tuning was used.
For G2T\_Trans\_TL, we first pre-trained the GE-EN parent model with a maximum of 100,000 updates, and then fine-tuned the model on the ASL-EN parallel dataset. We set the number of BPE merge operations to 20,000.
To register the ASL gloss to the source vocabulary, we appended the dummy parallel data with each ASL gloss paired with a NULL string to the training dataset for the parent model.

The results show that the BLEU sore of G2T\_Trans\_TL lagged far behind that of G2T\_PBSMT.
The bottom section of the table represents the pipeline model's results.
For the results obtained by PBSMT --- that is, the first part of the pipeline ---  we observed a trend similar to that of G2T\_PBSMT; that is, the best BLEU score was achieved when neither MERT nor PRO was used for weight tuning.
For the second stage of the pipeline, we evaluated the single-step transformer-based NMT pipelined with the best PBSMT, denoted by Model no.~5.
The results indicate that the pipeline model achieved a significantly higher BLEU score than G2T\_PBSMT.
We made the same observation for the results of the unigram chrF, which achieved the best scores for the pipeline model.

These results confirm that our proposed PBSMT + transformer pipeline model is also effective for the translation between ASL glosses and English sentences in an extremely low-resource setting, where the size of the parallel corpus is on the order of 10,000 words.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Ken Yano}{%
	Ken Yano is an invited researcher at the National Institute of Advanced Industrial Science and Technology (AIST). This paper was written when he was a specially appointed researcher at the University of Electro-Communications (UEC). He received his B.E. degree from Waseda University in 1992 and his Ph.D. degree from Hiroshima University in 2009. Before joining UEC, he worked as a post-doctoral researcher at Osaka University, Fukuoka University, ATR, NAIST, and Tokyo Institute of Technology.
From 1992 to 2007, he worked as a software engineer at several companies such as Tandem Computers, Openwave Systems, and Sharp Ltd.
}

\bioauthor[:]{Akira Utsumi}{%
Akira Utsumi is currently a Professor at the University of
Electro-Communications. He received his B.E. and Ph.D. degrees from
the University of Tokyo in 1990 and 1993, respectively. 
He was a Research Associate from 1993 to 1996, an Assistant Professor at the Tokyo Institute of Technology from 1996 to 2000,
and an Associate Professor at the University of Electro-Communications from 2000 to 2013. 
He was president of the Japanese Cognitive Science Society from 2021 to 2022. 
His research interests include the cognitive science of language and natural language processing.
}
\end{biography}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\biodate




\end{document}

