\documentclass[english]{jnlp_3.7}
\usepackage[T1,T5]{fontenc} % T5 for Vietnamese and T1 for English
\usepackage[vietnamese,japanese,english]{babel} % load both languages
\usepackage[main=english]{pxbabel}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{hangcaption_jnlp}
\usepackage{array}
%%%%\usepackage{udline}
%%%%\setulminsep{1.2ex}{0.2ex}
%%%%\let\underline=\ul
\allowdisplaybreaks


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
%%%%\usepackage{times}
%%%%\usepackage{amsmath}
%%%%\usepackage{url}
%%%%\renewcommand\UrlFont{\rmfamily}
%%%%\usepackage{times}
%%%%\usepackage{latexsym}
%%%%\usepackage{mathtools}
%%%%\usepackage{amssymb}
%%%%\usepackage{svg}
%%%%\usepackage{subfig}
%%%%\usepackage{pgfplots}
%%%%\pgfplotsset{compat=1.5}
\usepackage{pifont}
\newcommand{\cmark}{\ding{51}}%
\newcommand{\xmark}{\ding{55}}%
%%%%\usepackage{tabu} % use tabu with better compatibility with siunitx
    \usepackage{siunitx}
\usepackage{booktabs} % toprule
%%%%\usepackage{makecell} % cell verticle \rothead{}, \diaghead \eline{3}
\usepackage{multirow} % \multicolumn{2}{r}{hoge} \multirow{2}{*}{hoge}
%%%%\usepackage{colortbl}
\newcommand{\superscript}[1]{\ensuremath{^{\textrm{#1}}}} % e.g., \superscript{*}
\newcommand{\fn}[1]{\footnote{#1}}
%%%%\usepackage{url}
%%%%\usepackage{color}
%%%%\setlength{\fboxsep}{1pt} % small colorbox
%%%%\usepackage{xcolor}
    \usepackage[dvipdfmx,table]{xcolor}
\definecolor{mypurple}{RGB}{232,226,247}
\definecolor{myblue}{RGB}{0,113,188}
\definecolor{darkgreen}{rgb}{0.0, 0.2, 0.13}
\definecolor{ao(english)}{rgb}{0.0, 0.5, 0.0}
\definecolor{cornellred}{rgb}{0.7, 0.11, 0.11}
\newcommand{\Fig}[1]{Figure \ref{fig:#1}}
\newcommand{\Tab}[1]{Table \ref{tab:#1}}
\newcommand{\Eq}[1]{Eq. (\ref{eq:#1})}
\newcommand{\Algo}[1]{Algorithm \ref{algo:#1}}
\newcommand{\Sec}[1]{Section \ref{sec:#1}}
\newcommand{\ra}{{$\rightarrow$}}
\newcommand{\la}{{$\leftarrow$}}
\newcommand{\greenit}[1]{\textcolor{ao(english)}{\textbf{#1}}}
\newcommand{\redit}[1]{\textcolor{cornellred}{\textbf{#1}}}
\newcommand{\FIXME}[1]{\fbox{$\star$} (#1)}
%%%%\usepackage{soul}
%%%%\sethlcolor{blue}
%%%%\renewcommand{\hl}[2][blue]{\textcolor{#1}{#2}}
%%%%\newcommand{\mycomment}[1]{\textit{#1}}
%%%%\newcommand{\mycite}[1]{\cite{#1}}
%%%%\newcommand{\myciteA}[1]{\citeA{#1}}
%%%%\soulregister\mycite7
%%%%\soulregister\myciteA7
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{31}
\Number{1}
\Month{March}
\Year{2024}
%%%%\Issuetitle{Special Issue: XXXXXXXX}

\received{2023}{10}{19}
\accepted{2023}{11}{30}

\setcounter{page}{155}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{DiverSeg: Leveraging Diverse Segmentations with \\Cross-granularity Alignment for Neural Machine Translation}
\eauthor{Haiyue Song\affiref{KyotoU}\affiref{NICT} \and Zhuoyuan Mao\affiref{KyotoU} \and Raj Dabre\affiref{NICT} \and Chenhui Chu\affiref{KyotoU}  \and Sadao Kurohashi\affiref{KyotoU}\affiref{NII}} 
\eabstract{
In this study, we proposed DiverSeg to exploit diverse segmentations from multiple subword segmenters that capture the various perspectives of each word for neural machine translation. In DiverSeg, multiple segmentations are encoded using a subword lattice input, a subword-relation-aware attention mechanism integrates relations among subwords, and a cross-granularity embedding alignment objective enhances the similarity across different segmentations of a word. We conducted experiments on five datasets to evaluate the effectiveness of DiverSeg in improving machine translation quality. The results demonstrate that DiverSeg outperforms baseline methods by approximately two BLEU points. Additionally, we performed ablation studies to investigate the improvement over non-subword methods, the contribution of each component of DiverSeg, the choice of subword relations, the choice of similarity metrics in alignment loss,  and combinations of segmenters.
}
\ekeywords{Neural Machine Translation, Subword Segmentation, Subword Lattice, Subword-relation-aware Attention, Cross-granularity Alignment}
\headauthor{Song et al.}
\headtitle{Diverse Segmentations with Cross-granularity Alignment for NMT}

\affilabel{KyotoU}{}{Graduate School of Informatics, Kyoto University}
\affilabel{NICT}{}{National Institute of Information and Communications Technology}
\affilabel{NII}{}{National Institute of Informatics}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\selectlanguage{english}
\section{Introduction} %\label{sec:introduction}

A subword segmenter is a crucial preprocessing component in modern neural machine translation (NMT) models  \cite{1609.08144,2010.11125}.
The subword segmenter can address the out-of-vocabulary (OOV) problem \cite{sennrich-etal-2016-neural,kudo-richardson-2018-sentencepiece} as well as enhance the quality of translations compared with non-subword methods \cite{kambhatla-etal-2022-auxiliary,he-etal-2020-dynamic,song-etal-2022-bertseg}.



Each type of subword segmenter captures a unique aspect of the input text. The 
byte-pair encoding (BPE) segmenter \cite{sennrich-etal-2016-neural} based on the byte-pair encoding compression algorithm \cite{gage1994new} represents the input text in a highly compressed token sequence \linebreak
\cite{galle-2019-investigating}. The SentencePiece (SPM) subword segmenter, which is equipped with a unigram language model, captures contextual information to a certain extent \cite{kudo-richardson-2018-sentencepiece}. The neural subword segmenter BERTSeg \cite{song-etal-2022-bertseg} is tuned on pre-trained BERT models and enables the use of semantic meaning and the ability to segment words into linguistically motivated subwords. However, NMT models based on a single segmenter cannot capture various aspects of the input text.

To overcome this limitation, one approach is to use BPE segmenters of varying vocabulary sizes to generate multiple segmentations and input them into a lattice-aware transformer encoder \cite{xiao-etal-2019-lattice}. This method demonstrates that high diversity in segmentations can improve MT performance. However, this approach uses only one type of segmenter; thus, multi-view information of the input text cannot be captured effectively, and the relations between subwords are incomplete. Another approach uses BPE and SPM segmenters to generate more diverse segmentations for each sentence in the dataset \cite{kambhatla-etal-2022-auxiliary}, which is compatible with all NMT models because it performs only data-level augmentation. However, it uses the vanilla transformer model \cite{NIPS2017_3f5ee243} which is not aware of any connections between the different segmentations of one word.
Moreover, these two approaches lack explicit information sharing among multiple segmentations in the training objective, which results in the embeddings of multiple segmentations of the same word far apart in the embedding space.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1
\begin{figure}[b]
\begin{center}
\includegraphics{31-1ia6f1.pdf}
\end{center}
\hangcaption{Subword lattice and flattened lattice. The flattened version is suitable for the input of the transformer model where each subword is accompanied by ($start$, $end$) positional information.}
\label{fig:subwordlattice}
\vspace{-0.5\Cvs}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

In this study, we proposed DiverSeg to address these three problems using three key components: 1) a subword lattice input that contains character, BPE subwords, and BERTSeg subwords with positional information as shown in Figure \ref{fig:subwordlattice}; 2) a subword-relation-aware attention mechanism with fine-grained relation types calculated from the relative position information; 3) a cross-granularity alignment objective that maximizes the cosine similarity among various subword segmentations.

We evaluated the proposed DiverSeg approach on five datasets, including Asian Language Treebank (ALT) \cite{thu-etal-2016-introducing}, IWSLT'15 Vietnamese--English, WMT'16 Romanian--English, WMT'15 Finnish--English, and WMT'14 German--English. Experimental results demonstrate that the proposed approach outperformes baseline methods by approximately two BLEU points.
Furthermore, we performed ablation studies to examine the improvement over non-subword methods, contribution of each component, effect of different subword-relation mechanisms in the subword-relation-aware attention, effect of using various similarity measurement metrics in the cross-granularity embedding alignment loss, and selection of the combination of subword segmenters.
The contributions of this study are as follows.
\begin{itemize}
    \item DiverSeg, a method that leverages diverse segmentations from multiple subword segmenters (\Sec{method}), consists of a subword lattice input (\Sec{method1}), a subword-relation-aware attention mechanism (\Sec{method2}), and a cross-granularity embedding alignment objective (\Sec{method3}).
    \item Experimental results on five datasets that demonstrate the effectiveness of DiverSeg (\Sec{result1}).
    \item Analysis of how DiverSeg leads to performance improvements (Sections \ref{sec:result3}, \ref{sec:result4}, \ref{sec:result5}, and \ref{sec:result6}).
\end{itemize}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Work}

\noindent \textbf{Subword Segmenters.} \citeA{sennrich-etal-2016-neural} proposed BPE for subword segmentation to solve the OOV problem \cite{NIPS2014_5346}. Word Piece Model (WPM) \cite{6289079} is a method similar to BPE, but with the distinction that WPM selects the symbol pairs that maximize the likelihood of training data, whereas BPE selects the most frequent pairs. \citeA{kudo-richardson-2018-sentencepiece} proposed SentencePiece, which supports a unigram language model and can handle multiple languages, including those without whitespace word boundaries, such as \linebreak
Chinese or Japanese. Byte-level BPE (BBPE) \cite{1909.03341} is a variant of BPE that operates on bytes rather than on characters. In VOLT \cite{xu-etal-2021-vocabulary}, the optimal size of the subword vocabulary is calculated based on the optimal transport algorithm.
A survey provided information on non-neural segmentation methods and their limitations \cite{2112.10508}.
Dynamic programming encoding (DPE) \cite{he-etal-2020-dynamic} and bilingual subword segmentation \linebreak
\cite{deguchi-etal-2020-bilingual} have been proposed for NMT. In these methods, both target and source languages are considered. Bilingual subword segmentation is based on a unigram language model, whereas the DPE is based on a transformer model. The BERTSeg \cite{song-etal-2022-bertseg} and SelfSeg \cite{selfseg} segmenters obtain subword segmentations that maximize the generation likelihood, where they require only word-level monolingual data for training and inference, rendering the model highly efficient. In addition to general-purpose subword segmentation, task-specific subword segmentation methods \cite{hiraoka2021,hiraoka2022} aim to obtain appropriate tokenizations based on the loss of downstream tasks.

Stochastic subword segmentation is an efficient method for improving the robustness of the NMT models. BPE-dropout \cite{provilkov-etal-2020-bpe} is a stochastic version of BPE \cite{sennrich-etal-2016-neural} where the merge operations of subwords are randomly dropped during decoding, resulting in various segmentations for each word. SentencePiece regularization \cite{kudo-2018-subword} is a stochastic version of the SPM \cite{kudo-richardson-2018-sentencepiece}, which provides multiple segmentations with probabilities for each sentence from a unigram language model. However, these stochastic methods 
are based on BPE or SPM; therefore, they cannot capture linguistical information, such as BERTSeg \cite{song-etal-2022-bertseg} or DPE \cite{he-etal-2020-dynamic} does. The proposed DiverSeg framework combines information from multiple subword segmenters.

\noindent \textbf{Lattice Models.}
Lattice-aware models are widely used in numerous natural language processing (NLP) tasks such as speech translation \cite{zhang-etal-2019-lattice}, machine translation \cite{xiao-etal-2019-lattice}, language modeling \cite{buckman-neubig-2018-neural}, Chinese Named Entity Recognition (NER) \cite{li-etal-2020-flat}, and pre-trained BERT or GPT models \cite{lai-etal-2021-lattice,guo-etal-2021-lichee}. 
The manner in which models interpret the lattice inputs differ largely according to the target tasks. 
For the Chinese NER task, using relative position representations \cite{xiao-etal-2019-lattice}, which are the difference of positions between any two subwords, achieves comparable performance compared with using pre-defined subword relations \cite{li-etal-2020-flat}. However, for the MT task, using pre-defined subword relations, which are calculated using the start and end positions, can achieve a better performance than using the relative position representations \cite{xiao-etal-2019-lattice}. We improve from the coarse-grained seven types of relations \cite{xiao-etal-2019-lattice} to fine-grained $20$ types by 1) separating the many-to-one mappings, where multiple position relationships are mapped into one subword relation type, into one-to-one mappings, and 2) adding new types considering word boundary information. 



\noindent \textbf{NMT with Multiple Segmentations.} 
Hierarchical subword features based on BPE with various vocabulary sizes have been incorporated into RNN-based models \cite{morishita-etal-2018-improving}.
\citeA{kambhatla-etal-2022-auxiliary} proposed combining multiple subword segmentations in the training set by copying sentences segmented from various segmenters. \citeA{xiao-etal-2019-lattice} proposed using BPE with different vocabulary sizes to generate multiple segmentations and \citeA{takase-etal-2022-single} proposed leveraging multiple subword segmentations for subword-regularized models as a model ensemble approach.
In contrast to previous models, the proposed subword-relation-aware attention mechanism leverages multiple relation matrices to improve its ability of the model to capture complex subword-level relationships. The proposed cross-granularity alignment explicitly enhances the information sharing between different segmentations.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Preliminary}

This section provides an overview of the standard transformer-encoder architecture \cite{NIPS2017_3f5ee243} that is the basis for our model. The subsequent section details the components in DiverSeg.

Let $x = (x_1, x_2, …, x_n)$ be the input sequence of subword tokens, where $n$ is the sequence length. The embedding layer maps each token $x_i$ to a vector $\mathbf{e}_i \in \mathbb{R}^d$, where $d$ is the embedding dimension. The positional encoding (PE) layer \cite{NIPS2017_3f5ee243} adds a vector $\mathbf{p}_i \in \mathbb{R}^d$ to each embedding $\mathbf{e}_i$, where $\mathbf{p}_i$ is computed as follows:
\begin{subequations}\label{eq:1}
\begin{align}
    p_{i,2j} &= \sin\left(\frac{i}{10000^{2j/d}}\right) \\
    p_{i,2j+1} &= \cos\left(\frac{i}{10000^{2j/d}}\right) 
\end{align}
\end{subequations}
where $j = 0, 1, …, \frac{d}{2}-1$. The input of the transformer encoder is $\mathbf{Z}_0 = (\mathbf{z}_{0,1}, \mathbf{z}_{0,2}, …, \mathbf{z}_{0,n})$, where $\mathbf{z}_{0,i} = \mathbf{e}_i + \mathbf{p}_i$.

The transformer encoder consists of $L$ encoder layers with identical structures and non-sharing parameters, where each encoder layer consists of two sub-layers: a multi-head self-attention layer and a feed-forward layer. The multi-head self-attention layer computes the weighted sum of the input vectors, where the weights are determined by the similarity between the input vectors. The similarity is measured by the dot product of the input vectors and scaled by the square root of the dimension. The input vectors are also projected onto various subspaces using linear transformations to allow the model to attend to different aspects of the input. Each head in one layer performs the following computations:
\begin{align} 
    \mathbf{Q} &= \mathbf{Z}_{l-1}\mathbf{W}^\mathrm{Q}  \label{eq:2}\\
    \mathbf{K} &= \mathbf{Z}_{l-1}\mathbf{W}^\mathrm{K}  \label{eq:3}\\
    \mathbf{V} &= \mathbf{Z}_{l-1}\mathbf{W}^\mathrm{V}  \label{eq:4}\\
    \mathbf{A}_{\mathrm{weight}} &= \text{softmax}(\frac{\mathbf{Q}\mathbf{K}^\top}{\sqrt{d/h}}) \label{eq:5}\\
    \mathbf{A}_{\mathrm{value}} &= \mathbf{A}_{\mathrm{weight}} \mathbf{V} \label{eq:6}
\end{align}
where $h$ is the number of heads,  $d/h$ is the dimension of each head, $\mathbf{W}^\mathrm{Q},\mathbf{W}^\mathrm{K}$, and $\mathbf{W}^\mathrm{V} \in \mathbb{R}^{d \times d/h}$ are the projection matrices to generate the query, key, and value matrices $\mathbf{Q}, \mathbf{K}, \mathbf{V}\in \mathbb{R}^{n \times d/h}$. 

The output of the $l_{th}$ encoder layer $\mathbf{Z}_{l}\in \mathbb{R}^{n \times d}$ is then obtained by concatenating the $\mathbf{A}_{\mathrm{value}}$ of all heads and multiplying the output projection matrix $\mathbf{W}^\mathrm{O}\in \mathbb{R}^{d \times d}$ and fed into a feed-forward layer.
The output of the transformer encoder is $\mathbf{Z}_L$, where $L$ denotes the number of encoder layers. Each sub-layer is with residual connection \cite{He_2016_CVPR} and layer normalization \cite{1607.06450}. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Method} \label{sec:method}

DiverSeg contains three parts that differ from the conventional transformer model \cite{NIPS2017_3f5ee243}: 1) the subword lattice input with the subword-relation matrix represents diverse segmentations (\Sec{method1}), 2) the subword-relation-aware attention mechanism captures the relations between any two subwords (\Sec{method2}), and 3) the cross-granularity alignment objective explicitly aligns multiple segmentations (\Sec{method3}).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Multiple Segmentation Representations} \label{sec:method1}

We use diverse subword segmentation methods, including character-level segmentation, BPE \cite{sennrich-etal-2016-neural} and BERTSeg \cite{song-etal-2022-bertseg}, to generate multiple segmentations in DiverSeg and represent them by pre-defined $20$ fine-grained subword relation types, where previous work \cite{xiao-etal-2019-lattice} only used BPE segmenter and used seven coarse-grained subword relation types.

In the implementation of DiverSeg, we use \textit{absolute positional encoding} \cite{NIPS2017_3f5ee243} to encode the positional information of each subword and a variant of \textit{relative positional encoding} \cite{shaw-etal-2018-self} to encode the information between any two subwords, which is saved in a subword-relation matrix.

\noindent \textbf{Absolute Positional Encoding}. 
A subword lattice structure (\Fig{subwordlattice}) is used to represent a word with multiple segmentations. Each edge represents a subword in the lattice (formally, a directed acyclic graph). We convert the lattice structure into a sequence structure
to fit the input format of the transformer encoder. We flatten the lattice by concatenating multiple segmentations and assign each subword an absolute position denoted by its start position, which is calculated using \Eq{1}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 2
\begin{figure}[b]
\begin{center}
\includegraphics{31-1ia6f2.pdf}
\end{center}
\hangcaption{Model overview. 1) Segmentations from multiple segmenters are concatenated and accompanied by positions. 2) Relation matrices are generated from positions and provided to the subword-relation-aware attention mechanism. 3) Different segmentations of each word are aligned. Only two segmenters are presented for simplicity.}
\label{fig:overview}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\noindent \textbf{Relative Positional Encoding}. In addition to the start position, each subword carries two other types of positional information, the end position and word position, with all indices starting from $0$, as shown in \Fig{overview}.
\begin{itemize} 
    \item word position $p^\mathrm{w}_i$, the index of the word that the subword belongs to.
    \item start position $p^\mathrm{s}_i$, the index of the character that the subword starts with.
    \item end position $p^\mathrm{e}_i$, the index of the character that the subword ends with.
\end{itemize}

\noindent \textbf{Subword-relation Matrix} represents the relative relation between two subwords, which is generated from five simple types of relative positional relations of any two subword pairs.
\begin{itemize}
    \item word--word relation $r_{\mathrm{ww}}^{i,j}=\mathbf{ORF}(p^{\mathrm{w}}_i,p^{\mathrm{w}}_j)$
    \item start--start relation $r_{\mathrm{ss}}^{i,j}=\mathbf{ORF}(p^{\mathrm{s}}_i,p^{\mathrm{s}}_j)$
    \item start--end relation $r_{\mathrm{se}}^{i,j}=\mathbf{ORF}(p^{\mathrm{s}}_i,p^{\mathrm{e}}_j)$
    \item end--start relation $r_{\mathrm{es}}^{i,j}=\mathbf{ORF}(p^{\mathrm{e}}_i,p^{\mathrm{s}}_j)$
    \item end--end relation $r_{\mathrm{ee}}^{i,j}=\mathbf{ORF}(p^{\mathrm{e}}_i,p^{\mathrm{e}}_j)$
\end{itemize}
where the \textbf{o}rder relation function ($\mathbf{ORF(\cdot,\cdot)}$) returns the order relation of the first input to the second input $\in \{<,=,>\}$. We generate the subword relation of two subwords $x_i$ and $x_j$ through \Tab{relation-types}, which is a look-up table that maps five positional relations to one relation type.
We then obtain subword-relation matrix $\mathbf{R} \in \{1, ..., 20\}^{n\times n}$. Comparing with previous work \cite{xiao-etal-2019-lattice}, we involve word boundary information in \textbf{Type} $1$ and \textbf{Type} $20$. We also separate the coarse-grained many-to-one conditions into fine-grained one-to-one conditions. For instance, relation \textbf{inc} ($p^{\mathrm{s}}_i\le p^{\mathrm{s}}_j<p^{\mathrm{e}}_j\le p^{\mathrm{e}}_i$) in the previous work \cite{xiao-etal-2019-lattice} is separated into \textbf{Type} $6$, \textbf{Type} $7$, \textbf{Type} $10$, \textbf{Type} $11$ in our definition.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Subword-relation-aware Attention}  \label{sec:method2}

DiverSeg uses the same relation-aware self-attention architecture as in \citeA{shaw-etal-2018-self}. However, DiverSeg uses fine-grained subword relations as shown in \Tab{relation-types} whereas clipped relative position is used in \citeA{shaw-etal-2018-self} and coarse-grained subword relation is used in \citeA{xiao-etal-2019-lattice}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[b]
\input{06table01.tex}%
\hangcaption{Subword relations. Relation of two subwords is calculated by the orders of word index $p^\mathrm{w}$, start index $p^\mathrm{s}$, and end index $p^\mathrm{e}$ of two subwords $x_i$ and $x_j$.}
\label{tab:relation-types}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The proposed DiverSeg model leverages the subword-relation matrix $\mathbf{R}$ through a subword-relation-aware attention mechanism, as illustrated in \Fig{relattn}, where $\mathbf{R}$ provides an additional attention weight matrix $\mathbf{A}_{\mathrm{weight}}^{\mathrm{rel}}$ and value matrix $\mathbf{A}_{\mathrm{value}}^{\mathrm{rel}}$. The calculation in each head per layer is as follows:

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 3
\begin{figure}[t]
\begin{center}
\includegraphics{31-1ia6f3.pdf}
\end{center}
\hangcaption{Computational graph. The subword-relation-aware attention in each head where all calculation modules are colored. The graph is \textit{symmetric}, where the self-attention mechanism is on the left and the subword-relation-aware attention mechanism is on the right. Variables in the middle are generated jointly by two attention mechanisms. The embedding and dropout layers are omitted for simplicity.}
\label{fig:relattn}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\noindent \textbf{Dimensional Transformation.} To obtain input vectors with the required dimension in the attention head, we transform input $\mathbf{Z}_{l-1}$ to $\mathbf{Q}$, $\mathbf{K}$, $\mathbf{V}$ and $\mathbf{R}$ to $\mathbf{K}_\mathrm{R}$, $\mathbf{V}_\mathrm{R}$. \Eq{new4} maps the matrix of discrete types  $\mathbf{R}$ into a matrix in the continuous vecter space $\mathbf{R}_\mathrm{e} \in \mathbb{R}^{n\times n\times d}$. 
\begin{align}
    \mathbf{Q} &= \mathbf{Z}_{l-1}\mathbf{W}^\mathrm{Q} \\
    \mathbf{K} &= \mathbf{Z}_{l-1}\mathbf{W}^\mathrm{K}  \\
    \mathbf{V} &= \mathbf{Z}_{l-1}\mathbf{W}^\mathrm{V}  \\
    \mathbf{R}_\mathrm{e} &= \mathrm{Emb}(\mathbf{R}) \label{eq:new4}\\
    \mathbf{K}_\mathrm{R} &= \mathbf{R}_\mathrm{e}\,\mathbf{W}_{\mathrm{R}}^{\mathrm{K}} \\
    \mathbf{V}_\mathrm{R} &= \mathbf{R}_\mathrm{e}\,\mathbf{W}_{\mathrm{R}}^{\mathrm{V}}
\end{align}

\noindent \textbf{Attention Weight Matrix} 
The attention weight matrix  $\mathbf{A}_{\mathrm{weight}} \in \mathbb{R}^{n\times n}$ is from the self-attention weight and the subword-relation-aware attention weight, as expressed in \Eq{new9}. Here, $\mathbf{A}_{\mathrm{weight}}^{\mathrm{self}}$ is the conventional self-attention weight matrix. \Eq{new8} calculates the subword-relation-aware attention weight matrix given the query $\mathbf{Q}$ and subword-relation-aware key $\mathbf{K}_\mathrm{R}$. The calculation process differs from those in \Eq{new7} because the shapes of $\mathbf{K} \in \mathbb{R}^{n\times d/h}$ and $\mathbf{K}_\mathrm{R}\in \mathbb{R}^{n\times n\times d/h}$ differs. We used the Einstein summation ($\text{einsum}$) operation to sum over along the desired indices, where $\mathbf{A}_{\mathrm{weight}_{(i,j)}}^{\mathrm{rel}}=\sum_k\mathbf{Q}_{{(i,k)}}\mathbf{K}_{\mathrm{R}_{(i,j,k)}}$.
\begin{align}
    \mathbf{A}_{\mathrm{weight}}^{\mathrm{self}} &= \frac{\mathbf{Q}\mathbf{K}^\top}{\sqrt{d/h}} \label{eq:new7}\\
    \mathbf{A}_{\mathrm{weight}}^{\mathrm{rel}} &= \frac{\text{einsum}('ik, ijk \rightarrow ij', \mathbf{Q}, \mathbf{K}_\mathrm{R})}{\sqrt{d/h}} \label{eq:new8} \\
    \mathbf{A}_{\mathrm{weight}} &= \text{softmax}(\mathbf{A}_{\mathrm{weight}}^{\mathrm{self}} + \mathbf{A}_{\mathrm{weight}}^{\mathrm{rel}}) \label{eq:new9}
\end{align}

\noindent \textbf{Attention Value Matrix.} 
\Eq{new10} calculates the attention value $\mathbf{A}_{\mathrm{value}}$ from the self-attention value $\mathbf{A}_{\mathrm{weight}}\mathbf{V}$ and the subword-relation-aware value $\mathbf{A}_{\mathrm{value}}^{\mathrm{rel}}=\text{einsum}('ik, ikj \rightarrow ij', \mathbf{A}_{\mathrm{weight}}, \mathbf{V}_\mathrm{R})$, where each element is calculated by
$\mathbf{A}_{\mathrm{value}_{(i,j)}}^{\mathrm{rel}}=\sum_k\mathbf{A}_{\mathrm{weight}_{(i,k)}}\mathbf{V}_{\mathrm{R}_{(i,k,j)}}$. The order of indices differs from that in \Eq{new8}.
\begin{align}
    \mathbf{A}_{\mathrm{value}} &= \mathbf{A}_{\mathrm{weight}}\mathbf{V} + \text{einsum}('ik, ikj \rightarrow ij', \mathbf{A}_{\mathrm{weight}}, \mathbf{V}_\mathrm{R}) \label{eq:new10}
\end{align}

All heads in one layer share $\mathbf{W}_{\mathrm{R}}^{\mathrm{K}}\in \mathbb{R}^{d \times d/h}$ and $\mathbf{W}_{\mathrm{R}}^{\mathrm{V}}\in \mathbb{R}^{d \times d/h}$. Therefore, $\mathbf{K}_\mathrm{R} \in \mathbb{R}^{n \times n \times d/n}$, $\mathbf{V}_\mathrm{R} \in \mathbb{R}^{n \times n \times d/n}$,  and the subword-relation-aware relative attention weight matrix $\mathbf{RelAttn}_\mathrm{w}$ are shared across all heads. The remaining parts such as the concatenation of multiple heads, feed-forward, residual connection, and layer normalization keep the same as the traditional transformer encoder.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Cross-granularity Alignment Objective}  \label{sec:method3} % Embedding Alignment->Mutual Learning?

To address the lack of explicit information sharing among multiple segmentations in previous studies \cite{xiao-etal-2019-lattice,kambhatla-etal-2022-auxiliary},
we propose a cross-granularity embedding alignment loss $\mathcal{L}_{\mathrm{Align}}$ that enables information sharing among various types of segmentations of one word. 

As presented in \textit{1) Subword \& Positions} in \Fig{overview}, the input subword sequences $x$ are generated by concatenating segmentations from $s$ segmenters. Therefore, $s$ spans of consecutive subwords represent one segmentation of word $w$.
As shown in the \textit{3) Cross-granularity Alignment} in \Fig{overview}, the representation of one segmentation $\mathbf{E}$ is calculated by averaging the embeddings of the subwords in that span, where each subword embedding is from the last encoder layer.

We define $\mathcal{L}_{\mathrm{Align}}^{\mathrm{w}}$ for word $\mathrm{w}$ by the average of cosine similarities ($\cos\langle \cdot,\cdot \rangle$) between each two segmentation representations as follows:
\begin{equation}
    \mathcal{L}_{\mathrm{Align}}^{\mathrm{w}}=-\frac{1}{s^2}\sum_{i=1}^{\mathrm{s}}\sum_{j=1}^{\mathrm{s}}\cos\langle\mathbf{E}_{i},\mathbf{E}_{j}\rangle \label{eq:13}
\end{equation}
and $\mathcal{L}_{\mathrm{Align}}$ for one sentence containing $n$ words by summing the losses of each word $\mathrm{w}_k$:
\begin{equation}
    \mathcal{L}_{\mathrm{Align}}=\sum_{k=1}^{n}\mathcal{L}_{\mathrm{Align}}^{\mathrm{w}_k}\label{eq:14}
\end{equation}

The overall $\mathcal{L}$ can be expressed as a summation for the alignment loss and the cross-entropy loss of the translation task $\mathcal{L}_{\mathrm{CE}}$ with a weight ratio $\alpha$ with $1$ as the default setting:
\begin{equation} 
    \mathcal{L} = \mathcal{L}_{\mathrm{Align}} + \alpha\mathcal{L}_{\mathrm{CE}} \label{eq:total-loss} 
\end{equation}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 2
\begin{table}[b]
\input{06table02.tex}%
\hangcaption{Statistics of the MT datasets. Numbers indicate parallel sentences in each set of each dataset. En TTR and XX TTR indicate the type-token ratios for data in English data and data in the other language (averaged in ALT) in each dataset.}
\label{tab:datasets}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experiment Settings}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Datasets}

In the NMT experiments, we used the ALT \cite{thu-etal-2016-introducing}, IWSLT'15 English--Vietnamese (En--Vi), WMT'16 English--Romanian (En--Ro), WMT'15 English--Finnish (En--Fi), and WMT'14 English--German (En--De) datasets.
The details of the datasets are presented in \Tab{datasets}, which contains the sizes of the datasets and type-token ratios, calculated by $\frac{\#\text{word\ types}}{\#\text{words}}$.

ALT is a multi-way parallel dataset that contains data in English and multiple Asian languages. We experimented with Filipino (Fil), Indonesian (Id), Japanese (Ja), Malay (Ms), Vietnamese (Vi), and simplified Chinese (Zh). We used the ALT-standard-split tool\fn{Provided here: \url{www2.nict.go.jp/astrec-att/member/mutiyama/ALT}} to split the dataset into training, validation, and test sets. For the WMT'16 English--Romanian dataset, we performed normalization to the Romanian-side data and removed diacritics following previous work \cite{sennrich-etal-2016-edinburgh}. 
We prepared the WMT'14 English--German dataset using a data clean and normalization tool in Fairseq.\fn{\url{https://github.com/facebookresearch/fairseq/blob/main/examples/translation/prepare-wmt14en2de.sh}}

We performed tokenization on all data. We applied Juman++ \cite{tolmachev-etal-2018-juman} to Japanese data, the Stanford-tokenizer \cite{manning-etal-2014-stanford} to Chinese data, and the Moses tokenizer \cite{koehn-etal-2007-moses} to data in other languages.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{DiverSeg Settings}

DiverSeg leverages multiple segmenters on the source side. Because our focus was on the encoder side, we applied the widely used BPE segmenter to the data in the target language, including baselines.
For \textit{English\ra the other language} translation direction, DiverSeg uses the character-level segmenter, BPE, and BERTSeg, where the latter two are trained on English data in the training set of each dataset.
For \textit{the other language\ra English} translation direction, DiverSeg uses the character-level segmenter, BPE, and SentencePiece with a unigram language model, where the latter two are trained on data in the other language in the training set of each dataset. We used SentencePiece rather than BERTSeg in \textit{the other language\ra English} direction because BERTSeg, which is based on the English characterBERT currently only supports English. 

The BPE, BERTSeg, and SentencePiece segmenters share the same vocabulary in the same direction in each dataset. We limited the sizes of both the source and target vocabularies for all models to $8,000$, which are near the optimal sizes provided by the VOLT algorithm \cite{xu-etal-2021-vocabulary} shown in Appendix \ref{sec:appendix_vocab}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{NMT Settings}

We implemented DiverSeg using the fairseq framework.\fn{\url{https://github.com/facebookresearch/fairseq}} The model architectures for the ALT dataset were determined through parameter searching on the number of encoder layers, decoder layers, and attention heads. The optimal settings were six encoder layers, six decoder layers, and one attention head. For other datasets, we used a standard \texttt{Transformer Base} architecture with six encoder layers, six decoder layers, and eight attention heads. We used the default pre-normalization in fairseq instead of post-normalization in the original transformer paper \cite{NIPS2017_3f5ee243} because pre-normalization exhibits faster and more robust training performance \cite{2002.04745}. The dropout and attention dropout rates were set to $0.1$.

During training, each batch in one GPU contains $3,072$ tokens in the source language. We used eight GPUs in all experiments, resulting in approximately $25$k source tokens per training batch. We used the Adam optimizer \cite{1412.6980}, with $\beta_{1}=0.9$, $\beta_{2}=0.98$, and $\epsilon=10^{-9}$. We used warmup and decay strategy for the learning rate following \citeA{NIPS2017_3f5ee243}, with warmup steps of $4,000$, initial learning rate to $1.7*10^{-7}$ and learning rate to $0.0005$. We used label smoothing for the cross-entropy loss with $\epsilon_{ls}=0.1$ \cite{1512.00567}. We calculated the loss in the validation step after each epoch and applied early stopping when no improvement was observed after $10$ epochs.

During inference, we used the checkpoint with the best loss on the validation set. We used beam search with a beam size of $12$ and length penalty $\alpha=1.4$. We evaluated the translation quality through BLEU \cite{papineni-etal-2002-bleu}, BLEURT \cite{sellam-etal-2020-bleurt}, METEOR \cite{banerjee-lavie-2005-meteor} scores and performed significant test \cite{koehn-2004-statistical} against all baselines.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Baseline Settings}

We compared DiverSeg with three types of baseline methods: 1) models using data segmented by one type of subword segmenter, 2) models using data segmented by multiple types of segmenters, and 3) models using stochastic subword segmenter.


\noindent \textbf{Single-segmenter Methods} We compared the proposed model to the BPE \cite{sennrich-etal-2016-neural}, SentencePiece \cite{kudo-richardson-2018-sentencepiece}, VOLT \cite{xu-etal-2021-vocabulary}, DPE \cite{he-etal-2020-dynamic}, and BERTSeg \cite{song-etal-2022-bertseg} methods. BERTSeg and DPE were applied only to English data. 

\noindent \textbf{Multiple-segmenter Methods} We compared with 1) using multiple segmentations for each source sentence in the dataset \cite{kambhatla-etal-2022-auxiliary}, where we used character-level segmentation, BPE, and BERTSeg in \textit{English\ra the other language} direction and use character-level segmentation, BPE and SentencePiece in \textit{the other language\ra English} direction, keeping the same with DiverSeg. During inference, we tested the input sentences segmented by each segmenter and reported the average scores. Detailed results are reported in \Tab{alt-bleu-multisub} in Appendix \ref{sec:appendix_multisub}.
2) using the BPE segmenter with multiple vocabulary sizes, where we used character-level segmentation, BPE with vocabulary sizes of $8$k and $16$k \cite{xiao-etal-2019-lattice}.\fn{Because the code is not public \cite{xiao-etal-2019-lattice}, we implemented the method by ourselves, where we use richer subword relations from \Tab{relation-types} (except types $1$ and $20$, which require word boundary information) than in their paper.}


\noindent \textbf{Stochastic Subword Segmentation Method} We present results of the BPE-dropout method \cite{provilkov-etal-2020-bpe}. We applied the algorithm with a default dropout rate of $0.1$ to the training set of each dataset except for the high-resource WMT'14 En--De dataset, where the dropout was set to a lower value $0.05$ that achieved a better performance on the validation set.


We compared DiverSeg with each subword segmenter it used and with multiple-segmenter methods that used the same subword segmenters. Furthermore, we verified using BPE-dropout as one of the segmenters in DiverSeg. For this setting, we applied the combination of character-level segmentation and BPE-dropout. An analysis of using different segmenter combinations is presented in \Sec{result6}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experimental Results}

\Sec{result1} presents results of the translation quality evaluation, \Sec{result2} compares with non-subword methods, \Sec{result3} conducts an ablation study on the three proposed modules, \Sec{result4} explores various representations of subword relations, \Sec{result5} examines different alignment loss choices, and \Sec{result6} analyzes results from various segmenter combinations.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[b]
\input{06table03.tex}%
\hangcaption{BLEU results on the ALT dataset. The best result in each group is highlighted in \textbf{bold}. 
Statistical significance is assessed for each individual direction, with a significance level of $p<0.01$ denoted by $^{\clubsuit}$. 
We compared DiverSeg to the best-performing method in each direction and contrast DiverSeg w/ BPE-dropout against BPE-dropout. 
For the \textit{Avg.} score's statistical significance, we concatenated all translated files and compared them against the combined files of the \textit{one best} baseline method that had the highest \textit{Avg.} score.}
\label{tab:alt-bleu}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 4
\begin{table}[t]
\input{06table04.tex}%
\hangcaption{BLEU results on the IWSLT'15 and WMT datasets. The highest score in each group is highlighted in \textbf{bold}. Statistical significance is assessed for each individual direction, with a significance level of $p<0.05$ denoted by $^{\spadesuit}$. We compare DiverSeg to the best-performing method in each direction and also contrast DiverSeg w/ BPE-dropout with BPE-dropout.}
\label{tab:other-bleu}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Main Results} \label{sec:result1}

We compared DiverSeg with the single- and multiple-segmenter baselines. The BLEU scores of DiverSeg and the baselines are reported in Tables \ref{tab:alt-bleu} and \ref{tab:other-bleu}.
DiverSeg outperformed baseline models in most translation directions across all datasets. Specifically, when using the combination of character-level data, BPE and BERTSeg, DiverSeg, DiverSeg achieves $3.59$ and $4.52$ BLEU score compared with the best multiple-segmenter method \cite{xiao-etal-2019-lattice} and the BPE baseline, respectively, on average across the ALT dataset in the English\ra Asian languages translation directions. When using the combination of character-level data, BPE, and SentencePiece, the improvement brought by the DiverSeg becomes smaller than when involving BERTSeg, while still better than all baselines on the ALT dataset in the Asian languages\ra English translation direction. For the MultiSub method \cite{kambhatla-etal-2022-auxiliary}, we report the average scores using input sentences segmented by each segmenter and provide a detailed analysis in \Tab{alt-bleu-multisub} in Appendix \ref{sec:appendix_multisub}.
However, the improvement of DiverSeg over other methods on larger datasets, such as WMT'15 En--Fi and WMT'14 En-De, was not as significant as that on small datasets. The gap between all the segmentation methods decreased for large datasets. We hypothesized that this phenomenon could occur because multiple segmentation methods increase the number of times each subword is trained as a data augmentation method, and on large datasets, each subword is trained sufficiently. 
Additionally, we observed that for source languages with higher type-token ratios (as shown in \Tab{datasets}), DiverSeg yielded more significant BLEU score improvements. For example, for the IWSLT'15 En--Vi dataset, where the type-token ratio of the English data is higher, we observe a more considerable improvement in the En\ra Vi direction. For the WMT'15 En--Fi dataset, we can observe a more considerable improvement in the Fi\ra En direction because Finnish is a morphologically rich language, and the type-token ratio is higher than that of English.

We also present the results of using the stochastic subword segmentation method BPE-dropout \cite{provilkov-etal-2020-bpe}, as well as the stochastic version of DiverSeg, where we incorporated both character-level segmentation and BPE-dropout segmentation. In low-resource scenarios, BPE-dropout showed impressive performance, and DiverSeg with BPE-dropout showed even better results, achieving +$3$ BLEU scores. Conversely, in high-resource scenarios, the performances of the two stochastic methods were comparable. 
We abstained from experiments on the WMT14 dataset because the BPE-dropout method did not surpass BPE, and we were constrained by computational resources.

We used truecase data during both training and inference. When we converted the generated output and reference to lowercase, the BLEU scores for BPE-dropout reached $26.20$ and $32.50$ for the En\ra De and De\ra En directions, respectively. These results are close to those reported in the BPE-dropout study \cite{provilkov-etal-2020-bpe}, which used lowercase data for both training and inference. Although they provided results using truecase data for small datasets, they did not report the results using truecase data for large datasets. Moreover, our results indicated a negligible improvement in both directions of the WMT'14 En--De dataset when contrasting BPE-dropout with BPE.
\citeA{provilkov-etal-2020-bpe} reported similar results for a larger dataset with $16$M lowercase training samples. We hypothesized that when using truecase data, the threshold for the amount of training data decreases. This is based on the result that improvement decreases when using truecase data, as shown in the Appendix of \citeA{provilkov-etal-2020-bpe}.
The BLEURT results are presented in \Tab{bleurt}, and the METEOR results are presented in \Tab{meteor}, where a similar tendency to the BLEU results can be observed.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Comparison with Non-subword Methods} \label{sec:result2}

We evaluated DiverSeg against non-subword approaches on the IWSLT'15 En--Vi and \linebreak
WMT'16 En--Ro datasets. 
As presented in \Tab{ablation-nonsubword}, the results indicate that the proposed method considerably outperforms non-subword approaches in terms of the BLEU scores. This finding is consistent with previous research, which has shown that subword methods such as BPE and SentencePiece often outperform character- or word-level approaches for source sentences \cite{1711.04457}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 5
\begin{table}[p]
\input{06table05.tex}%
\caption{BLEURT results. The best result in each group is in highlighted in \textbf{bold}.}
\label{tab:bleurt}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 6
\begin{table}[p]
\input{06table06.tex}%
\caption{METEOR results. The best result in each group is in highlighted in \textbf{bold}.}
\label{tab:meteor}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 7
\begin{table}[p]
\input{06table07.tex}%
\caption{Comparison to non-subword methods. We report BLEU scores, and the best result is in \textbf{bold}.}
\label{tab:ablation-nonsubword}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Ablation Study of Proposed Modules} \label{sec:result3}

\Tab{ablation-parts} present results of an ablation study assessing the effectiveness 
\pagebreak
of the three proposed modules: 1) the lattice positional encoding (PE) that provides absolute positional information; 2) the subword-relation-aware attention mechanism that provides relative positional information; 3) the cross-granularity embedding alignment objective $\mathcal{L}_{\mathrm{Align}}$. Comparison of ID $1$ to $2$, $3$, and $5$ indicates that each individual module contributes to the final result. Comparison of ID $5$ and $6$ reveals that $\mathcal{L}_{\mathrm{Align}}$ is essential for the relative PE and significantly increases the BLEU score by $0.74$. Additionally, at least one of the absolute position or the relative position is required (ID $7$ and $8$), otherwise, the model input degenerates into a bag-of-words input, leading to low performance.
All ablation studies were conducted in the IWSLT'15 En\ra Vi direction, which is representative because of the moderate size of IWSLT'15.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 8
\begin{table}[t]
\input{06table08.tex}%
\hangcaption{Ablation of components. \greenit{\cmark} indicates using the component and \redit{\xmark} indicates not using it. Tested on the En\ra Vi direction of the IWSLT'15 dataset and the best result is in \textbf{bold}.
Default settings of DiverSeg are marked in \colorbox{mypurple}{purple}.}
\label{tab:ablation-parts}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Choice of Representing Subword Relations} \label{sec:result4}

We compared the choice of representing subword relations with previous methods: 
%%%%1) \citeA{xiao-etal-2019-lattice} did not utilize word information $r_{\mathrm{ww}}$,\fn{We use the remaining $18$ types of relations instead of $7$ types of relations in their paper.} 2) used $the$ distance to represent the relative position between two subwords \cite{shaw-etal-2018-self}, and 3) did not use relative PE. 
1) without utilizing word information $r_{\mathrm{ww}}$ \cite{xiao-etal-2019-lattice},\fn{We use the remaining $18$ types of relations instead of $7$ types of relations in their paper.} 2) utilizing distance to represent the relative position between two subwords \cite{shaw-etal-2018-self}, and 3) traditional absolute PE without using relative PE (Vaswani et al. 2017).

\Tab{ablation-attn} shows that using relative information can substantially improve all three metrics compared to the model without relative PE. 
Moreover, using pre-defined relations like word boundary information and finer-grained relations (e.g., adjacency) in our proposed model outperformed the distance-based method proposed by \citeA{shaw-etal-2018-self}. Word boundary information improves all three metrics.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 9
\begin{table}[b]
\input{06table09.tex}%
\caption{Ablation of subword relations. Test on the En\ra Vi direction of the IWSLT'15 dataset.}
\label{tab:ablation-attn}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Choice of Metrics and Weight Ratio in Alignment Loss} \label{sec:result5}

We examined the metrics and weight ratios used to align various segmentations of one word. In addition to maximizing the cosine similarity of the embeddings as expressed in \Eq{13}, we attempted to minimize the L1 or L2 distance. We also examined various weight ratios as shown in \Eq{total-loss}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 10
\begin{table}[b]
\input{06table10.tex}%
\caption{Ablation of similarity measurements. Test on IWSLT'15 En\ra Vi. The best result is in \textbf{bold}.}
\label{tab:ablation-loss}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 4
\begin{figure}[b]
\begin{center}
\includegraphics{31-1ia6f4.pdf}
\end{center}
\hangcaption{Embedding shifting visualization. The shifting of hidden representations by \textbf{left}: maximizing cosine similarity, and \textbf{right}: minimizing L2 distance in the cross-granularity alignment loss function. }
\label{fig:ablation-loss}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The results of using various metrics are presented in \Tab{ablation-loss}, which reveals that minimizing the distance causes the training to fail. 
The reason for this is illustrated in \Fig{ablation-loss}: \textbf{1)} During maximizing the cosine similarity, the norms of embeddings remain the same; \textbf{2)} however, while minimizing the L1 or L2 distance, the embeddings drift toward the origin because embeddings with small norms tend to have a small distance. We alleviated this problem by embedding normalization $\mathbf{E}_\mathrm{n}=\frac{\mathbf{E}}{\mathrm{norm}(\mathbf{E})}$ and labeled them as Norm L1 and Norm L2 distances. The normed versions exhibited superior performance; however, their performance was poorer than those using cosine similarity. This result can be attributed to the ``curse of dimensionality,'' where the distance between any two embeddings tends to be similar and their differences cannot be captured meaningfully \cite{Steinbach2004}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 5
\begin{figure}[b]
\begin{center}
\includegraphics{31-1ia6f5.pdf}
\end{center}
\caption{Ablation of alignment loss weight ratios. Test on IWSLT'15 En\ra Vi.}
\label{fig:loss_weight_ratio}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The results of using various weight ratios are presented in \Fig{loss_weight_ratio}, showing ratio as $1$ or $2$ is the optimal setting for the IWSLT'15 En\ra Vi direction.
We can observe that it is an unimodal function in terms of BLUE score where setting the weight ratio too low or too high negatively impacts performance.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 11
\begin{table}[b]
\input{06table11.tex}%
\hangcaption{Ablation of segmenter combinations. Test on the En\ra Vi direction of the IWSLT'15 dataset. The best result among each group is in \textbf{bold}.}
\label{tab:ablation-seg}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Choice of Segmenter Combinations} \label{sec:result6}

\Tab{ablation-seg} presents a comparison of the various combinations of segmenters. 
\pagebreak
The combination of character-level, BPE, and BERTSeg delivered the highest translation performance, as indicated by the BLEU score within the \textit{Non-stochastic Method} group. Furthermore, the combination of character-level and BPE-dropout outperformed the combination of character-level, BPE, and BERTSeg.  
Based on this finding, we suggest not combining dynamic segmentation with fixed segmentation because dynamic segmentation may cover fixed segmentation in certain epochs, except for character-level segmentation which provides positional information.

Moreover, the diversity of the segmenters in a combination is a crucial factor. 
We define the segmentation difference rate using the proportion of words in the corpus that are segmented differently by two segmenters, where each type of word counts only once. \Fig{segsimilarity} illustrates that the difference rates between BPE and BERTSeg, and between BPE and SentencePiece were high. Therefore, combining BPE and BERTSeg (SentencePiece) provides diverse perspectives of the word in the source language and a dense input lattice, resulting in excellent translation quality. We found a low difference rate between BPE and DPE, which may explain the low translation quality when combining them. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 6
\begin{figure}[t]
\begin{center}
\includegraphics{31-1ia6f6.pdf}
\end{center}
\hangcaption{Segmentation difference rate.  Numbers indicate the percentage of words where two segmenters provided distinct segmentations.}
\label{fig:segsimilarity}
\vspace{-0.5\Cvs}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Analysis}

In this section, we present several analyses that investigate the effectiveness of DiverSeg.
We provide translated examples in Section \ref{sec:translatedexamples}, introduce a new metric called target word accuracy in Section \ref{sec:tgtwordacc}, visualize the attention mechanism in Section \ref{sec:attnvis}, and visualize embedding in Section \ref{sec:embvis} to gain insights into how DiverSeg improves translation quality.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Translated Examples} \label{sec:translatedexamples}

We present an example in \Tab{case-study} and demonstrate how DiverSeg produces the correct translation as evidenced by the attention heatmap in \Fig{attn_heatmap}. 
In the example, the Vietnamese word \foreignlanguage{vietnamese}{``nói''} should be translated into ``speak'' in English as in the reference. However, both BPE and BERTSeg produced incorrect translations ``say'' and ``talk,'' respectively. The translation generated by BERTSeg exhibited a higher semantic similarity with the reference, and only DiverSeg correctly translated the word. 
The attention heatmap in Figure \ref{fig:attn_heatmap} reveals that when generating the correct English word ``speak,'' DiverSeg attends to various segmentations of the corresponding Vietnamese word \foreignlanguage{vietnamese}{``tập nói''} (learn to speak).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 12
\begin{table}[b]
\input{06table12.tex}%
\hangcaption{Translation example. The proposed DiverSeg method correctly translates the source word to the target word ``speak.'' From the IWSLT'15 Vi--En dataset.}
\label{tab:case-study}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 7
\begin{figure}[b]
\begin{center}
\includegraphics{31-1ia6f7.pdf}
\end{center}
\caption{Attention heatmap. In the DiverSeg model from the target sentence to the source sentence.}
\label{fig:attn_heatmap}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Target Word Accuracy with Different Frequency Ranges} \label{sec:tgtwordacc}

\Tab{wordfreq} presents a comparison of the accuracy of translating the target words with varying frequencies.
Accuracy was calculated by extracting each Vietnamese word from the reference sentence, determining its frequency in the training set, and evaluating whether it appeared in the translation. The proposed DiverSeg outperformed all other methods in almost all frequency bins, demonstrating its effectiveness and robustness. However, although the character-based NMT model achieved high accuracy for low-frequency words, primarily because of its copying ability, DiverSeg did not completely integrate this ability. Information from BERTSeg interferes with the generation process, whereas BERTSeg performs poorly on low-frequency target words but well on high-frequency words.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 13
\begin{table}[t]
\input{06table13.tex}%
\hangcaption{Word translation accuracy. Translation accuracy comparison on target words of varying frequency evaluated in the IWSLT'15 En\ra Vi translation direction.}
\label{tab:wordfreq}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Visualization of Subword-relation-aware Attention} \label{sec:attnvis}

We demonstrated the roles of self-attention and subword-relation-aware attention through visualization using the BertViz tool \cite{vig-2019-multiscale}.
The self-attention was extracted using \Eq{new7} and subword-relation-aware attention was extracted using \Eq{new8} from the last layer of the transformer encoder and averaged for all heads. 
As illustrated in \Fig{analysis-attn}, in contrast to self-attention, which attends to all subwords, subword-relation-aware attention can overlook information from subwords belonging to multiple segmentations of the same word, indicating its ability to identify word boundaries.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 8
\begin{figure}[t]
\begin{center}
\includegraphics{31-1ia6f8.pdf}
\end{center}
\hangcaption{Attention visualization. Two examples showing the difference between the self-attention mechanism and the proposed subword-relation-aware attention mechanism.} 
\label{fig:analysis-attn}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Visualization of Segmentation Embeddings} \label{sec:embvis}

We investigated the effectiveness of the cross-granularity embedding alignment objective by comparing the word embeddings DiverSeg models with and without the alignment objective (ID $2$ in \Tab{ablation-parts}). We generated the English word embeddings from a randomly selected sentence in the IWSLT'15 En\ra Vi direction during inference. 
Each point in \Fig{analysis-emb} represents the word embeddings calculated by averaging the embeddings of the subwords it contains. Subsequently, the subword embeddings were extracted from the final encoder layer. We only preserved words containing more than one BPE or BERTSeg subwords in the sentence and applied t-SNE  \cite{tsne} and principal component analysis (PCA) methods to map them to 2D points.
The results revealed that with the alignment objective, different segmentations of a word tended to remain close, whereas without the alignment objective, the embeddings of the same word obtained by different segmentation methods were widely dispersed.  
For the t-SNE results, we observed that with alignment, only three word embeddings calculated from character-level segmentation (Cameron, walking, and client) were not close to the word embeddings calculated from the corresponding BPE and BERTSeg segmentations. 
In the without-alignment method, word embeddings from character-level segmentation form an isolated cluster. This is more pronounced in PCA results, where, without alignment, embeddings from character-level segmentations are entirely distinct from other embeddings.
This finding provides evidence that the proposed cross-granularity alignment objective successfully aligned the semantic meanings of multiple subword segmentations.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 9
\begin{figure}[t]
\begin{center}
\includegraphics{31-1ia6f9.pdf}
\end{center}
\hangcaption{Word embedding visualization of one sentence using t-SNE (above) and PCA (below), where the left column shows embeddings with the cross-granularity alignment objective and the right column shows embeddings without the alignment objective. Word embedding is obtained by averaging all subword embeddings in that word, where $\mid$ represents the subword boundary and colors represent the segmentation method for that embedding.}
\label{fig:analysis-emb}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Efficiency} \label{sec:efficiency}

\noindent \textbf{Time Efficiency.}  
Methods that use a single segmenter take approximately $0.44$ minutes \linebreak
to complete one epoch of training. This includes subword-based methods, such as BPE, \linebreak
SentencePiece, and BERTSeg. The character-based method takes $1.56$ minutes for each epoch, primarily because of the longer input sequence.
The method that employs multiple segmenters based on \texttt{Transformer Base} model \cite{kambhatla-etal-2022-auxiliary} requires an average of \mbox{5.67 minutes} to complete one epoch owning to the use of more data. The method of \citeA{xiao-etal-2019-lattice} takes approximately $12.38$ minutes,\footnote{This is presumably because the training speed of their model is $0.46$ times that of the base model.} while  
DiverSeg takes $16.56$ minutes per epoch. This demonstrates a moderate efficiency reduction when compared to other methods that utilize multiple segmentations.


\noindent \textbf{Parameters.} 
The model parameters are listed in \Tab{para}. DiverSeg is parameter efficient, where it uses only $6\%$ more parameters compared with the model using BPE segmentation. Among the $2.3M$ extra parameters, only $0.2M$ parameters are from the subword-relation-aware attention module, and $2.1M$ parameters are from the larger look-up table in the encoder embedding layer because it covers both the BPE and BERTSeg vocabularies. Among the multi-segmenter methods, DiverSeg is almost as parameter efficient as the data mixing method \cite{kambhatla-etal-2022-auxiliary}. In contrast, there are more parameters in \citeA{xiao-etal-2019-lattice} because the look-up table covers more tokens in multiple BPE vocabularies, whose size is up to $16k$.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 14
\begin{table}[t]
\input{06table14.tex}%
\hangcaption{Model parameters. Obtained by actual measurement from models in the IWSLT'15 En\ra Vi translation direction.}
\label{tab:para}
\vspace{-0.5\Cvs}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion and Future Work}

In this study, we introduced DiverSeg to leverage diverse subword segmentations and capture the multiple perspectives of each input sentence. DiverSeg contains a lattice input module, a subword-relation-aware attention mechanism, and a cross-granularity embedding alignment objective. We conducted extensive experiments on multiple machine translation datasets, and the results demonstrated that DiverSeg can be used to effectively exploit the potential of multiple segmentations to improve NMT performance. Ablation studies and analyses were performed to evaluate the contribution of each proposed module.

In the future, we will focus on designing a decoder architecture to enable learning from target sentences with multiple subword segmentations during training and exploring a decoding algorithm that combines the probabilities of multiple subword paths of a single word during inference. Additionally, the training speed is a limitation of DiverSeg. One potential method to expedite the training process is to optimize the computation of the cross-granularity loss.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment

This study was supported by JSPS KAKENHI Grant Number JP21J23124. Part of this work was conducted during an internship at NICT. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%\bibliographystyle{jnlpbbl_1.7}
    \bibliographystyle{jnlpbbl_1.7B}
\selectlanguage{english}
\bibliography{06refs}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\appendix

\vspace{-0.75\Cvs}
\section{Vocabulary Size Selection} \label{sec:appendix_vocab}
The optimal vocabulary sizes calculated by the VOLT algorithm \cite{xu-etal-2021-vocabulary} are presented in \Tab{volt-size}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 15
\begin{table}[b]
\vspace{-0.5\Cvs}
\input{06table15.tex}%
\caption{Optimal BPE vocabulary sizes of languages in each dataset.}
\label{tab:volt-size}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 16
\begin{table}[t]
\input{06table16.tex}%
\caption{BLEU results on the ALT dataset using input by each segmenter in the MultiSub method.}
\label{tab:alt-bleu-multisub}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Detailed Results of MultiSub}\label{sec:appendix_multisub}

During training, MultiSub \cite{kambhatla-etal-2022-auxiliary} leverages character-level data, BPE segmentation, and BERTSeg segmentation in the \textit{English\ra the other language} direction and character-level data, BPE segmentation, and SentencePiece segmentation in \textit{the other language\ra English} direction. During inference, we test the performance using the sentence segmented by every single segmenter. As presented in \Tab{alt-bleu-multisub}, using the character-level input data show achieved a very low performance, whereas using BPE or BERTSeg improved performance. 

DiverSeg enhanced the connection between the three segmentations through the subword-relation-aware attention mechanism in the model and the cross-granularity embedding alignment during the training process. Therefore, the performance was considerably higher than using the same three segmenters. 



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Haiyue Song}{%
Haiyue Song received his B.S. in Computer Science from Shanghai Jiao Tong University in 2018, and his M.S. in Informatics from Kyoto University in 2020. He is currently a Ph.D. student at Kyoto University and a technical researcher in NICT, Japan. His research interests include natural language processing, in particular neural machine translation and subword segmentation.}


\bioauthor[:]{Zhuoyuan Mao}{%
Zhuoyuan Mao received his B.S. degree in Mathematics from East China University of Science and Technology, Shanghai, China, in 2017 and the M.S. degree in Informatics from Kyoto University, Kyoto, Japan, in 2021, where he is currently pursuing a Ph.D. His research interests include natural language processing, machine translation, and sentence representation learning.
}

\bioauthor[:]{Raj Dabre}{%
Raj Dabre received his M.Tech. in Informatics from IIT Bombay (2014) and his Ph.D. in Informatics from Kyoto University (2018). He is currently a senior researcher in NICT, Japan, and a visiting researcher in AI4Bharat. His research interests include natural language processing, multimodality, efficiency and multilingualism.
}

\bioauthor[:]{Chenhui Chu}{%
Chenhui Chu received his B.S. in software engineering from  Chongqing University in 2008, and his M.S. and Ph.D. in Informatics from Kyoto University in 2012 and 2015, respectively. He is currently a program-specific associate professor at Kyoto University. His research interests include natural language processing, particularly machine translation and multimodal machine learning. 
}

\bioauthor[:]{Sadao Kurohashi}{%
Sadao Kurohashi received a PhD in Electrical Engineering from Kyoto University in 1994. He is currently the director-general of the National Institute of Informatics, Japan, and a specially appointed professor of the Graduate School of Informatics at Kyoto University. His research interests include natural language processing, knowledge infrastructure, and open science. He received the 10th and 20th anniversary best paper awards from Journal of Natural Language Processing in 2004 and 2014 respectively, 2009 IBM faculty award, and 2010 NTT DOCOMO mobile science award, and 2017 Commendation for Science and Technology by the Minister of Education.
}

\end{biography}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\biodate



\end{document}

