\documentclass[english]{jnlp_3.8}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{array}
\allowdisplaybreaks

\usepackage{hangcaption_jnlp}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition

\usepackage[T1]{fontenc}

%%%%%\usepackage{graphicx}
%%%%%\usepackage{xspace,mfirstuc,tabulary}
%%%%%\usepackage[most]{tcolorbox}
%%%%%\usepackage{graphicx}
%%%%%\usepackage{algorithm}
%%%%%\usepackage{url}
%%%%%\usepackage{amsthm,amsmath,amssymb}
\usepackage{xcolor}
%%%%%\usepackage{listings}
%%%%%\usepackage{tikz}
%%%%%\usepackage{bm}
\usepackage{booktabs}
%%%%%\usepackage{array}
\usepackage{arydshln}
%%%%%\usepackage{ulem}


\definecolor{forestgreen}{rgb}{0.13, 0.55, 0.13}

%%%%%\newcommand{\seq}[1]{\boldsymbol{#1}}
%%%%%\usepackage{color} 
%%%%%\definecolor{mypink2}{RGB}{219, 48, 122}
%%%%%\definecolor{orange}{RGB}{255, 147, 00}
%%%%%\definecolor{jrcolor}{RGB}{100, 150, 225}
%%%%%\definecolor{jrcomment}{RGB}{70, 200, 150}
\definecolor{grey}{RGB}{166, 166, 166}

%%%%%\newcommand{\PredSty}[1]{\textnormal{\ttfamily\color{mygreen!90!black}#1}\unskip}
%%%%%\newcommand{\Xmat}[0]{{{\bf X}}}
%%%%%\definecolor{mygreen}{HTML}{3cb44b}

%%%%%\newtcolorbox{AIbox}[2][]{aibox,title=#2,#1}

%%%%%\tcbset{
%%%%%  aibox/.style={
%%%%%    width=450pt,
%%%%%    top=10pt,
%%%%%    % colback=white,
%%%%%    colframe=black,
%%%%%    colbacktitle=black,
%%%%%    enhanced,
%%%%%    center,
%%%%%    attach boxed title to top left={yshift=-0.1in,xshift=0.15in},
%%%%%    boxed title style={boxrule=0pt,colframe=white,},
%%%%%  }
%%%%%}
%%%%%\input ulem.sty\relax
%%%%%\catcode`\@=11 %

%%%%%\font\uwavefontold=lasy6
%%%%%\font\uwavefont=lasyb10 scaled 652

%%%%%\def\uwave{%
%%%%%  \bgroup
%%%%%    \markoverwith{%
%%%%%      \lower3.5\p@\hbox{\uwavefontold\char58}%
%%%%%    }%
%%%%%  \ULon
%%%%%}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{32}
\Number{3}
\Month{September}
\Year{2025}
%%%%\Issuetitle{「　　　　」特集号}


\received{2024}{12}{25}
\revised{2025}{5}{5}
\accepted{2025}{6}{20}


\setcounter{page}{886}


%%%%   Title, Author, etc.            %%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Cross-lingual Contextualized Phrase Retrieval}
\eauthor{Huayang Li\affiref{naist} \and Deng Cai\affiref{tencent} \and Zhi Qu\affiref{naist} \and Qu Cui\affiref{tencent} \and \\ Hidetaka Kamigaito\affiref{naist} \and Lemao Liu\affiref{tencent} \and Taro Watanabe\affiref{naist}
} 
\eabstract{%
Phrase-level dense retrieval has shown many appealing characteristics in downstream NLP tasks by leveraging the fine-grained information that phrases offer. In our work, we propose a new task formulation of dense retrieval,  \textit{cross-lingual contextualized phrase retrieval}, which aims to augment cross-lingual applications by addressing polysemy using context information. However, the lack of specific training data and models are the primary challenges to achieve our goal. As a result, we extract pairs of cross-lingual phrases using word alignment information automatically induced from parallel sentences. Subsequently, we train our \textbf{C}ross-lingual \textbf{C}ontextualized \textbf{P}hrase \textbf{R}etriever (CCPR) using contrastive learning, which encourages the hidden representations of phrases with similar contexts and semantics to align closely. Comprehensive experiments on both the cross-lingual phrase retrieval task and a downstream task, i.e, machine translation, demonstrate the effectiveness of \textsc{CCPR}. On the phrase retrieval task, \textsc{CCPR} surpasses baselines by a significant margin, achieving a top-1 accuracy that is at least 13 points higher. When utilizing \textsc{CCPR} to augment the large-language-model-based translator, it achieves average gains of 0.7 and 1.5 in BERTScore for translations from X$\Rightarrow$En and vice versa, respectively, where X is one of the six Indo-European languages in the WMT16 dataset. It also achieves gains of at least 0.7 BERTScore on translations into East Asian languages, such as Japanese, Chinese, and Korean.
}

\ekeywords{Retrieval-Augmented Generation, Cross-lingual Phrase Retrieval, Large Language Model}
\headauthor{Li et al.}
\headtitle{Cross-lingual Contextualized Phrase Retrieval}



%%%%     Affiliation     %%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\affilabel{naist}{}{Nara Institute of Science and Technology}
\affilabel{tencent}{}{Tencent AI Lab}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle
\renewcommand{\thefootnote}{\fnsymbol{footnote}}
\footnote[0]{This paper is an extended version of our preliminary paper \cite{li-etal-2024-cross-lingual} presented in Findings of the Association for Computational Linguistics: EMNLP 2024. We revised the main paper to include more experiments on East Asian languages, analyses, details, and discussions in the experimental sections.
}
\renewcommand{\thefootnote}{\arabic{footnote}}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S1
\section{Introduction}

Compared with the dense retrieval at sentence (or passage) level \cite{karpukhin-etal-2020-dense,pmlr-v162-borgeaud22a,asai2024selfrag}, learning dense retrieval at the phrase level has shown more appealing characteristics in extensive NLP tasks, such as entity linking \cite{gillick-etal-2019-learning}, open-domain question answering \cite{lee-etal-2021-learning-dense,lee-etal-2021-phrase}, text generation \cite{lan2023copy,cao2024retrieval}, etc. An important reason is that phrases can provide more fine-grained information than sentences.


In cross-lingual research, the phrase-level dense retrieval also shows the promise to solve a range of NLP problems \cite{bapna-firat-2019-non,zheng-etal-2022-cross-lingual}. The cross-lingual phrase retrieval in \citeA{zheng-etal-2022-cross-lingual} focuses on the mapping of cross-lingual Wiki entities, where each of them is represented by averaged hidden representations from 32 different contexts. However, the constrained phrase type, i.e., the wiki entity, makes the cross-lingual phrase retriever difficult to augment general NLP tasks. Moreover, unlike wiki entities, which may have fewer ambiguities, general-type phrases that are lexically identical can have different meanings depending on their contexts. Thus, accounting for polysemy \cite{cruse1986lexical} using the context information becomes critical.
Even when the lexically identical phrases share similar semantics, a more appropriate context would provide richer information for utilizing the phrases in downstream tasks \cite{min-etal-2019-discrete,cao2024retrieval}. 



Therefore, we advocate for a new task formulation, i.e., \textit{cross-lingual contextualized phrase retrieval}, which aims to find the cross-lingual phrase that is mostly relevant to the provided (general-type) source phrases, considering their meanings and surrounding contexts. However, achieving this goal is non-trivial, due to the scarcity of specific training data and models.
Since annotating cross-lingual contextualized phrase pairs of general type is very difficult and expensive, we first introduce a data collection method, which leverages the automatically induced word alignment information from parallel sentences to extract suitable cross-lingual phrase pairs for training. This ensures that the phrases are of general type and both cross-lingual phrases and contexts are well aligned. Thereafter, we propose a \textbf{C}ross-lingual \textbf{C}ontextualized  Phrase Retriever (CCPR) which is trained on the constructed dataset. The \textsc{CCPR} mainly employs phrase-level contrastive learning to draw cross-lingual phrases with similar contexts and meanings closer in the hidden space. At inference time, we can leverage the well-trained CCPR to build a phrase-level index and use the query phases for search. 

Unlike previous works \cite{zheng-etal-2022-cross-lingual}, which focus solely on the task of cross-lingual phrase retrieval for wiki entities, our work also explores the potential of leveraging CCPR to augment downstream cross-lingual tasks, e.g., machine translation (MT). One critical question in front of us is how to select meaningful phrases for indexing and searching at inference time. To address this problem, we propose to learn a phrase segmentation module, which can be used to predict meaningful phrases from sentence- or passage-level retrieval data at inference time. The developed phrase segmentation module is important for ensuring the train-test consistency of CCPR.




Experiments on both of the cross-lingual contextualized phrase retrieval and a downstream task, i.e., MT, show the effectiveness of \textsc{CCPR}. We first evaluate \textsc{CCPR} on the cross-lingual contextualized phrase retrieval task, since a higher accuracy is more beneficial for downstream tasks, such as MT, cross-lingual dictionary induction, etc \cite{zhang2016bridging,sogaard-etal-2018-limitations}. 
For this task, we use the human annotated cross-lingual phrase pairs as the golden truth and evaluate whether \textsc{CCPR} and other baselines can retrieve those golden phrases from a large-scale index. Experiments show that \textsc{CCPR} outperforms the baselines for at least 13 points in terms of the top-1 accuracy. We also conduct evaluation on the MT task using a large language model (LLM), e.g., LLama-2 \cite{touvron2023llama}. This task aims to evaluate whether the information retrieved by \textsc{CCPR} can enhance LLM's ability of cross-lingual generation. Following the fashion of retrieval-augmented generation (RAG) \cite{lewis2020retrieval}, we simply integrate the retrieved phrase information to the input of the LLM, and compare our method with other baselines. 
For both X$\Rightarrow$En and En$\Rightarrow$X translation directions on the WMT16 dataset, where X is from six languages, our \textsc{CCPR} achieves averaged gains of 0.7 and 1.5 BERTScore points \cite{zhang2019bertscore}, respectively. Results on other evaluation metrics, e.g., COMET \cite{rei-etal-2022-comet}, are consistent with BERTScore.

In summary, our contributions are three-fold:
%%%%%
\begin{itemize}
    \item We propose a new formulation of dense retrieval, i.e., \textit{the cross-lingual contextualized phrase retrieval}, which has substantial potential in augmenting cross-lingual tasks.
    \item   We propose a \textbf{C}ross-lingual \textbf{C}ontextualized \textbf{P}hrase \textbf{R}etriever (\textsc{CCPR}), which uses the constructed training data, i.e., cross-lingual phrases extracted from automatically induced word alignment information, to learn both the phrase alignment and segmentation modules.  
    \item  We conduct extensive experiments on cross-lingual contextualized phrase retrieval and MT. Experiments show that our method outperforms the baselines by a large margin. 
\end{itemize}
%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2
\section{Related Work}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2.1
\subsection{Dense Retrieval} 
In text generation, dense retrieval has been widely studied at both sentence and phrase levels. Particularly after the emergence of large language models (LLMs), how to retrieve related sentences (or passages) \cite{izacard2021contriever,ni-etal-2022-sentence} for input to LLMs has became a popular research topic, namely, retrieval-augmented generation (RAG) \cite{lewis2020retrieval,karpukhin-etal-2020-dense,guu2020retrieval,pmlr-v162-borgeaud22a,asai2024selfrag}.  However, compared to the sentence-level retrieval, phrase-level retrieval has been shown to not only enhance retrieval accuracy but also markedly boost performance in downstream tasks \cite{lee-etal-2021-learning-dense,lee-etal-2021-phrase}. By indexing and integrating retrieved phrases into the output of language models, \citeA{lan2023copy} and \citeA{cao2024retrieval} make the text generation process more attributable and accurate. These studies underline the effectiveness of phrase-level dense retrieval. Our work notably diverges from those works, because we focus more on cross-lingual tasks. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2.2
\subsection{Retrieval in Cross-lingual Tasks} 

In cross-lingual tasks, the majority of research has concentrated on sentence-level retrieval.  
Many research works in retrieval-augmented MT \cite{zhang-etal-2018-guiding,Gu2018SearchEG,xia2019graph,he-etal-2021-fast} have explored retrieving bilingual sentences based on similarity of source sentences, guiding MT models to utilize retrieved target information. A notable limitation is their reliance on bilingual data for constructing the index. This requirement substantially limits the scale and diversity of data available for retrieval. In contrast, \citeA{cai-etal-2021-neural} propose to directly retrieve relevant target sentences using source sentences to assist MT, facilitating cross-lingual retrieval using a monolingual index. Unlike the task-specific model for MT in \citeA{cai-etal-2021-neural}, many studies in dense retrieval aim to enhance the cross-lingual sentence retrieval on more languages and at a larger scale \cite{chidambaram-etal-2019-learning,conneau-etal-2020-unsupervised,reimers-gurevych-2020-making,heffernan-etal-2022-bitext,feng-etal-2022-language,cai-etal-2022-retrofitting}. 

However, cross-lingual phrase retrieval has not been extensively investigated.  \citeA{bapna-firat-2019-non} utilize the source side of the bilingual data for building a retrieval index and employ the similarity between source phrases for retrieval. Therefore, their indexing also suffers from the limitation of reliance on bilingual data.  In addition, it uses $n$-grams as phrases by default, leading to plenty of meaningless and noisy spans in the index. \citeA{zheng-etal-2022-cross-lingual} limit their cross-lingual phrase retrieval task to a specific phrase type, i.e., the wiki entity. The setting in their work overlooks the nuanced meanings that lexically identical phrases may convey in different contexts, making the retriever difficult to augment general NLP tasks. In contrast, our research aims to develop a cross-lingual contextualized phrase retriever for general-type phrases.

Additionally, there is research focused on employing nearest neighbor retrieval to support MT \cite{khandelwal2021nearest,zheng-etal-2021-adaptive,meng-etal-2022-fast,deguchi-etal-2023-subset}. These efforts differ from the aforementioned studies as they aim to estimate prediction distributions through nearest neighbor retrieval at each translation step.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3
\section{Task Formulation \label{sec:task}}
This work aims a cross-lingual phrase retriever that can augment downstream NLP tasks. To ensure its effectiveness, the retriever must adhere to two essential criteria. First, it should ensure that phrases in the index are of general type. Second, it must  resolve the polysemy of general-type phrases by considering their contextual differences.

Thus, we introduce a new task formulation, \textit{cross-lingual contextualized phrase retrieval}. We are given a large collection of $N$ general-type phrases $\mathcal{P}_{index} = \{\boldsymbol{p}_1, \dots, \boldsymbol{p}_{N}\}$ in language $L_y$, and a general-type query phrase $\boldsymbol{q}$ in language $L_x$. Each phrase, by default, is associated with the context information $(\boldsymbol{c}, s, e)$, where $\boldsymbol{c}$ represents the original sentence containing this phrase, and $s$ and $e$ are its start and end positions in $\boldsymbol{c}$. The primary objective is to identify a cross-lingual phrase $\boldsymbol{p} \in \mathcal{P}_{index}$ that is relevant to $\boldsymbol{q}$, considering their contexts and meanings. Our formulation notably diverges from the one proposed by \citeA{zheng-etal-2022-cross-lingual}, which focuses on a restricted type of phrases, i.e., wiki entities, ignoring the nuanced meanings that general-type phrases may convey in different contexts. 

This new task formulation also introduces several challenges, with the primary issue being the lack of training data. Specifically, there is a scarcity of cross-lingual phrase pairs of general type and accompanied by contextual information. Such data are crucial for training the model to recognize cross-lingual phrases that have similar contexts and meanings, allowing it to align these phrases closely in a hidden space. 
In addition, how to extract meaningful general-type phrases from sentences is also an open question, which is critical for employing the retriever to downstream tasks.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4
\section{Training Data Collection \label{sec:train_data}}

Annotating cross-lingual contextualized phrase pairs of general type is very difficult and expensive, posing a significant obstacle to training effective cross-lingual contextualized phrase retriever. However, the sentence-level parallel data in general domains are more readily available \cite{ng-etal-2019-facebook}. Additionally, as the lower unit of phrase alignment, the word alignment has been extensivly studied \cite{brown-etal-1993-mathematics,och-ney-2003-systematic,dyer-etal-2013-simple,li-etal-2019-word,jalili-sabet-etal-2020-simalign,dou-neubig-2021-word,wu-etal-2023-wspalign}.  Therefore, to overcome the data scarcity, we propose to use a word alignment model, such as GIZA++ \cite{och-ney-2003-systematic} and neural aligner \cite{dou-neubig-2021-word}, in order to automatically induce word alignments from parallel sentences and subsequently extract cross-lingual phrase pairs. This method allows us to produce phrase pairs that are not only of general type but also accompanied with contexts, aligning with our task requirements.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F1
\begin{figure}[b]
\begin{center}
\includegraphics{32-3ia5f01.eps}
\end{center}
    \hangcaption{(a) Cross-lingual phrase alignment takes the cross-lingual phrase pairs within similar contexts as positives and other in-batch phrases as negatives. Encoders on left and right sides employ different dropout mask $\boldsymbol{z}$, following Gao et al. (2021). The inputs of the dual encoders are parallel sentences. (b) A linear phrase-segmentation head is used to predict whether a span is a phrase or not. (c) The pipeline of using our learned model for downstream tasks. \label{fig:illustration}}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


Formally, given a pair of parallel sentences $\boldsymbol{x} = \{x_1, x_2, ..., x_{|\boldsymbol{x}|}\}$, and $\boldsymbol{y} = \{y_1, y_2, ..., y_{|\boldsymbol{y}|}\}$, where $|\boldsymbol{x}|$ and $|\boldsymbol{y}|$ are the number of words in the sequences, we use a word alignment model to obtain the word alignment information of $\boldsymbol{x}$ and $\boldsymbol{y}$ \cite{koehn-etal-2003-statistical,och-etal-1999-improved}. If every word in a consecutive span $\boldsymbol{x}_{i:j}$ can be aligned to a consecutive span $\boldsymbol{y}_{u:v}$, we will use them as a pair of aligned phrases $\boldsymbol{p}^{x}=\boldsymbol{x}_{i:j}$ and $\boldsymbol{p}^{y}=\boldsymbol{y}_{u:v}$, where $1\leq i\leq j \leq |\boldsymbol{x}|$ and $1\leq u\leq v \leq |\boldsymbol{y}|$. Notably, phrases that meet the constraints may range from a single word to the whole sequence, and extracted phrases are allowed to have overlaps. In addition, the context information of $\boldsymbol{p}^{x}$ and $\boldsymbol{p}^{y}$ are $(\boldsymbol{x}, i, j)$ and  $(\boldsymbol{y}, u, v)$, respectively. Unlike the phrase table extracted in SMT \cite{koehn-etal-2003-statistical,chiang-2005-hierarchical}, which is static and context-independent, here the phrase mapping between $\boldsymbol{p}^{x}$ and $\boldsymbol{p}^{y}$ is unique considering their context.

As shown in Figure \ref{fig:illustration} (a), The German phrase (``einer'', ``Schweigeminute'') is aligned with the the English phrase (``a'', ``minute'', ``'s'', ``silence'') within similar contexts, i.e., parallel sentences. More details about the extraction of cross-lingual contextualized phrase pairs are in Section \ref{sec:impl}. 



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5
\section{Methodology }

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.1
\subsection{Model Architecture \label{sec:alignment}}
We introduce a \textbf{C}ross-lingual \textbf{C}ontextualized \textbf{P}hrase \textbf{R}etriever (CCPR) for our new formulated task. The main target of CCPR is to make the representations of cross-lingual phrases with similar contexts and meanings to be close in the hidden space. To this end, we propose a \textit{cross-lingual phrase alignment} module based on contrastive learning, utilizing the data collected in Section \ref{sec:train_data} for training. In addition, one remaining problem is how to select phrases for indexing at inference time. To address this, we propose a \textit{phrase segmentation} module that uses phrase representations to predict meaningful phrases from sentences or paragraphs. At inference time, we can leverage the learned phrase segmentation module to select phrases for indexing, ensuring the train-test consistency.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.1.1
\subsubsection{Phrase encoding}
Our phrase encoder is based on a context encoder and an $\mathrm{MLP}$ layer. 
The encoding of a phrase $\boldsymbol{p}=\boldsymbol{x}_{s:e}$, is defined as:
\begin{align}
    \mathbf{H}^{z} &= \mathrm{ContextEncoder}(\boldsymbol{x}, \boldsymbol{z}) \nonumber \\
    \boldsymbol{h}^{z}_{p} &= \mathrm{MLP}_{align}\big([\mathbf{H}^z_{s}; \mathbf{H}^z_{e}]\big) \label{eq:enc}
\end{align}
where $\mathrm{ContextEncoder}(\boldsymbol{x}, \boldsymbol{z})$ is a Transformer model, e.g., BERT \cite{devlin-etal-2019-bert,feng-etal-2022-language} and RoBERTa \cite{conneau-etal-2020-unsupervised}, that encodes the context $\boldsymbol{x}$ to a matrix $\mathbf{H}^{z} \in \mathbb{R}^{N\times d}$ with a dropout mask $\boldsymbol{z}$. Inspired by previous works \cite{lan2023copy,lee-etal-2021-learning-dense,seo-etal-2018-phrase}, we use the concatenation of $\mathbf{H}^{z}_{s}$ and $\mathbf{H}^{z}_{e}$, i.e., the start and end token of the phrase, to represent phrase $\boldsymbol{x}_{s:e}$.
$\mathrm{MLP}_{align}(\cdot)$ is a function that maps the concatenation of two hidden states $\mathbf{H}^{z}_{s}$ and $\mathbf{H}^{z}_{e}$ from $\mathbb{R}^{2d}$ to $\mathbb{R}^{o}$, where $o$ is the output hidden size. The encoding for phrases in context $\boldsymbol{y}$ is similar.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.1.2
\subsubsection{Cross-lingual phrase alignment}
For a batch of parallel sentences, we can extract a collection of cross-lingual phrase pairs $\mathcal{P}_{pair} = \{(\boldsymbol{p}^{x}_{i}, \boldsymbol{p}^{y}_{i})\}_{i=1}^{K}$, where $\boldsymbol{p}^{x}_{i}$ and $\boldsymbol{p}^{y}_{i}$ are the positive examples of each other. Since each positive phrase pair is from parallel sentences that share similar semantics, directly learning on them may cause the model to learn a trivial shortcut. Therefore, inspired by SimCSE \cite{gao-etal-2021-simcse}, we apply two independently sampled dropout mask $\boldsymbol{z}$ and $\boldsymbol{z}^\prime$, to encode phrases $\boldsymbol{p}^{x}_{i}$ and $\boldsymbol{p}^{y}_{i}$, respectively, following Eq. \ref{eq:enc}. The training objective for the $x\rightarrow y$ direction is:

\begin{align}
    \mathcal{L}_{x\rightarrow y} &= -\frac{1}{K}\sum_{i=1}^K\log\frac{\exp(\boldsymbol{h}^{z}_{p^{x}_i}\cdot \boldsymbol{h}^{z'}_{p^{y}_i})}{Z_x(i)}, \\
   Z_x(i) &= \sum_{(p^x_j, p^y_j) \in \mathcal{P}_{pair}}\exp(\boldsymbol{h}^{z}_{p^{x}_i}\cdot \boldsymbol{h}^{z}_{p^{y}_j}) \label{eq:con_deno} 
\end{align}
where we use all the non-paired phrases in $\mathcal{P}_{pair}$ as in-batch negatives in Eq. \ref{eq:con_deno}. For bidirectional symmetry, our final loss is:

\begin{equation}
    \mathcal{L}_{align} = \mathcal{L}_{x\rightarrow y} + \mathcal{L}_{y\rightarrow x}.
\end{equation}

The illustration of the contrastive learning is shown in Figure \ref{fig:illustration} (a). In practice, we may use parallel data from multiple languages during training, i.e., the source contexts $\boldsymbol{x}$ in a batch may come from multiple languages, e.g., Germany, Czech, etc, which is inspired by the success of multi-lingual training in \citeA{liu-etal-2020-multilingual-denoising}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.1.3
\subsubsection{Phrase segmentation\label{sec:seg}}
This module aims to learn how to select phrases from sentences or passages, since most retrieval data at inference time comes in sentences or passages, rather than phrases.
To ensure the train-test consistency, we take the phrases extracted in Section \ref{sec:train_data} as positive data, and all the other spans in the corresponding contexts as negative data. Our phrase segmentation module is a binary classifier defined as follows:
\begin{align}
    P(T=1) = \sigma\Big(\mathrm{MLP}_{seg}\big([\mathbf{H}^z_{i}; \mathbf{H}^z_{j}]\big)\Big)\label{eq:pseg} 
\end{align}
where $T$ is the label for the span $\boldsymbol{x}_{i:j}$, where $1\leq i\leq j \leq |\boldsymbol{x}|$, $\sigma(\cdot)$ is an activation function, $\mathrm{MLP}_{seg}$ is a linear layer that maps the hidden representation from $\mathbb{R}^{2d}$ to $\mathbb{R}$, and the definitions for $\mathbf{H}^z_{i}$ and $\mathbf{H}^z_{j}$ are the same as in Eq. \ref{eq:enc}. We use the label $T=1$ for phrases and $T=0$ for non-phrase spans. The phrase segmentation for sequence $\boldsymbol{y}$ is similar to Eq. \ref{eq:pseg}. In practice, the number of non-phrase spans is significantly more than the number of phrases. To mitigate data imbalance, we employ a strategy of randomly sampling an equal number of non-phrase spans and phrases within a sentence during training \cite{li2020empirical}. The illustration of phrase segmentation is in Figure \ref{fig:illustration} (b). 

The training loss for the phrase segmentation is
\begin{align}
    \mathcal{L}_{seg} = -\frac{1}{|\mathcal{S}|}\sum_{\boldsymbol{p} \in \mathcal{S}} \Big( T_{p}\log P(T_{p}=1) + (1-T_p)\log\big(1-P(T_{p}=1)\big) \Big) \nonumber
\end{align}
where $\mathcal{S}$ is a set of spans, which are extracted from sequence $\boldsymbol{x}$ or $\boldsymbol{y}$. The learning objective of the whole model becomes:

\begin{equation}
    \mathcal{L} =  \mathcal{L}_{align} + \beta \mathcal{L}_{seg} 
\end{equation}
where $\beta$ is a hyper-parameter. It is worth noting that the parameters of $\mathrm{ContextEncoder}$ used for phrase alignment and segmentation are shared.

In our preliminary studies, we investigated various phrase segmentation strategies, including $n$-gram segmentation and learning a Byte Pair Encoding (BPE) model \cite{sennrich-etal-2016-neural,kudo-2018-subword} across word boundaries. However, these methods  struggle to identify meaningful phrases, leading to significant discrepancies between training and testing phases and performance declines.
In Section\ref{sec:mt}, we evaluate the effect of the learned phrase segmentation on MT task.




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.2
\subsection{Inference Pipeline\label{sec:inference}}
The pipeline of employing our cross-lingual contextualized phrase retriever for downstream NLP tasks involves two main steps: index building and searching, as shown in Figure \ref{fig:illustration} (c).

A significant benefit of our retriever is its ability to leverage monolingual data in the target language, eliminating the need for bilingual data to construct the index at inference time \cite{zhang-etal-2018-guiding,Gu2018SearchEG,bapna-firat-2019-non}. Given the retrieval data in target language, our retriever begins by segmenting  sentences (or passages) into phrases. When segmenting a sentence (or passage), we will enumerate all the possible spans within it and calculate their probabilities of being  meaningful phrases, according to Eq. \ref{eq:pseg}. We select all the spans whose probabilities are larger than a threshold for indexing. It is worth noting that overlapping phrases are allowed at the phrase segmentation time. The selected phrases are encoded into hidden representations according to Eq. \ref{eq:enc}, which will be used to construct a dense-retrieval index.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F2
\begin{figure}[b]
{\vskip -15pt}
\begin{center}
\includegraphics{32-3ia5f02.eps}
\end{center}
\hangcaption{An example instruction for a large-language-model based translator augmented by our method. The segmented phrases in the source sentence are in \textcolor{blue}{blue text} with underline. The retrieved translation of the phrases are in \textcolor{forestgreen}{green}, and their appearances in the contexts are marked by ``\textcolor{forestgreen}{[[]]}''. The \textcolor{grey}{text}  after ``English:'' is the reference, which is for illustration and will not appear in the instruction.}
 \label{tab:ours_prompt}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


For querying, we first process the query sentence (or paragraph) in a source language similarly, i.e., segmenting and encoding. Subsequently, we use the encoded query phrases to directly search for the related cross-lingual phrases using Maximum Inner Product Search (MIPS).  Because of the advanced data structure and search algorithm \cite{malkov2018efficient,douze2024faiss,johnson-2022-binary}, the search step is highly efficient (more details in Section \ref{sec:ccpr_exp}). The retrieved information, including the target phrase, its accompanied context, and its positions in the context, can then be integrated to downstream tasks to improve the performance. As shown in Figure \ref{tab:ours_prompt},  the retrieved phrases along with their surrounding contexts are integrated into the instructions given to an LLM for tasks such as MT.  



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6
\section{Experiments}
We propose to evaluate our model on cross-lingual contextualized phrase retrieval and MT.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6.1
\subsection{Implementation Details \label{sec:impl}}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6.1.1
\subsubsection{Training data collection}

We use all the bilingual training datasets of WMT16 on Huggingface\footnote{\url{https://huggingface.co/datasets/wmt16}} to train our \textbf{C}ross-lingual \textbf{C}ontextualized \textbf{P}hrase \textbf{R}etriever (CCPR). The WMT16 dataset has six language pairs, inclduing De-En, Cs-En, Fi-En, Ru-En, and Tr-En. Because of the efficiency and the satisfactory performance of GIZA++ \cite{och-ney-2003-systematic,dou-neubig-2021-word}, we first use the GIZA++ software,\footnote{\url{https://github.com/moses-smt/mgiza}} i.e., IBM-4 model \cite{brown-etal-1993-mathematics}, to induce the word alignment for each pair of parallel sentences, and then extract the cross-lingual phrase pairs.  Notably, we did not induce the phrase table as in Phrase-based Statistical Machine Translation (PBSMT) \cite{koehn-etal-2003-statistical}. In contrast, in our setting, we extract the cross-lingual consecutive spans that are aligned in a pair of parallel sentence as cross-lingual phrase pairs. Each pair of the cross-lingual phrases is associated with the surrounding contexts. We filter out phrases that begin or end with words whose frequency is more than 30k in the training data. We also filter out  phrases that contain only numbers and punctuations. The final training dataset contains 1.3 billion pairs of cross-lingual phrases extracted from 10 million parallel sentences from six language pairs.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6.1.2
\subsubsection{Model}
In our work, we train two variants of our  model, \textsc{CCPR-XLMR} and \textsc{CCPR-LaBSE}, whose parameters are initialized from  XLM-RoBERTa-base (\textsc{XLMR}) \cite{conneau-etal-2020-unsupervised} and \textsc{LaBSE} \cite{feng-etal-2022-language}, respectively. The hidden size $d$ of both models is 768. The output $o$ of $\mathrm{MLP}_{align}: \mathbf{R}^{2d} \rightarrow \mathbf{R}^{o}$ in Eq. (\ref{eq:enc}) is 128, which plays a critical role in reducing the memory cost of our phrase index while maintaining comparable performance.
Each model is trained on the mixture of all language pairs of WMT16. We train our model on 8 V100 GPUs for 20K steps, where the learning rate is 5e-5, dropout rate is 0.2, batch size on each device is 64. The $\beta$ for the phrase segmentation loss in Eq. \ref{eq:pseg} is $1$. Those hyper-parameters are tuned on the validation dataset. At inference time, we use the FAISS library\footnote{\url{https://github.com/facebookresearch/faiss}} \cite{douze2024faiss} to build our retrieval index. We use the \texttt{FlatIP} as our index type. 



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6.2
\subsection{Cross-lingual Contextualized Phrase Retrieval \label{sec:ccpr_exp}}
As described in Section \ref{sec:task}, given a query phrase and its context in source language, the objective of this task is to identify the most relevant cross-lingual phrase from a large-scale index in target language, considering the context information.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6.2.1
\subsubsection{Setup} 

For this experiment, we build the test set based on the human annotated word alignment data \cite{jalili-sabet-etal-2020-simalign} and evaluate the retrieval accuracy of different methods.  

We process the word alignment data on three language pairs, De$\Rightarrow$En \cite{ghader-monz-2017-attention}, Ro$\Rightarrow$En \cite{mihalcea-pedersen-2003-evaluation}, and Cs$\Rightarrow$En \cite{marevcek2011automatic}. For the Cs$\Rightarrow$En dataset, we only leverage the data under the ``pcedt'' split, which are data from The Wall Street Journal (WSJ). For each pair of languages, we instruct human annotators to identify 200 pairs of high-quality phrases. More concretely, we first use the heuristic rules discussed in Section \ref{sec:impl} to collect an initial set of cross-lingual phrase pairs, and then ask the human annotators to answer two questions for each phrase pair:
\begin{enumerate}
    \item If the phrase is a single word, is it informative in the context?
    \item If the phrase has multiple words, is the semantics of this phrase complete and informative? For instance, ``local authorities and large'' is not a phrase with complete semantics. In addition, ``Of course'' has complete semantics but is not informative.
\end{enumerate}
If the answer to any questions is true, then we add this phrase to a pool of high-quality phrases. For those phrases, overlaps are allowed. Finally, we randomly sampled 200 annotated phrases from the pool for each language.

We compare our models, \textsc{CCPR-XLMR} and \textsc{CCPR-LaBSE}, with several baselines. The first two baselines \textsc{XLMR} \cite{conneau-etal-2020-unsupervised}  and \textsc{mBERT} \cite{devlin-etal-2019-bert}\footnote{\url{https://huggingface.co/models}} are models trained in multilingual data at the document level, without using the alignment information. We use them to encode the context and represent the phrases by concatenating the hidden representations of their start and end tokens, following the standard in prior works \cite{lan2023copy,lee-etal-2021-learning-dense,seo-etal-2018-phrase}. The second two baselines \textsc{mUSE} \cite{chidambaram-etal-2019-learning} and \textsc{LaBSE} \cite{feng-etal-2022-language} are trained on cross-lingual data with alignment information, and we use the model weights released by SBERT\footnote{\url{https://www.sbert.net}} \cite{reimers-gurevych-2020-making} to encode the contexts. To evaluate the effect of context-awareness, we propose two Non-contextualized Phrase Retrievers (NPR), i.e., \textsc{NPR-XLMR} and \textsc{NPR-LaBSE}. Compared with our contextualized models, the only difference is that the \textsc{NPR-X} methods use cross-lingual phrase pairs in semantically different contexts as positive examples, similar to the setup in \citeA{zheng-etal-2022-cross-lingual}. 

To mimic real-world conditions, where an index generally contains extensive extra data that are irrelevant to the user query, we use 9.6 million English phrases sampled from the WMT16 training dataset as the extra data, ensuring the retrieval process is non-trivial. We will build an independent index for each model. In this task, we use the top-1 accuracy to evaluate the performance of our methods and baselines.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6.2.2
\subsubsection{Results}
Table \ref{tab:retrival} shows that our models significantly outperform baselines in cross-lingual contextualized phrase retrieval, highlighting the capability of our models to accurately identify relevant cross-lingual phrases while being context-aware.  In contrast, the \textsc{NPR-LaBSE} and \textsc{NPR-XLMR}, i.e., the context-independent baselines inspired by \citeA{zheng-etal-2022-cross-lingual}, perform notably worse. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T1
\begin{table}[b]
\input{05table01.tex}
\hangcaption{Cross-lingual contextualized phrase retrieval. We use the accuracy@1 as our evaluation metric. The \textsc{CCPR} and \textsc{NPR} denote contextualized and non-contextualized cross-lingual phrase retriever, respectively. The Best results are highlighted in \textbf{bold text}. \label{tab:retrival}}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


In addition, we also evaluate the efficiency of our method. Under the setting of this experiment, it takes only 0.04 seconds to retrieve 32 most relevant cross-lingual phrases for 1,000 source phrases on 4 V100 GPUs, thanks to FAISS's high parallelism on GPUs \cite{douze2024faiss}. We believe that this retrieval latency is adequate for most real-time retrieval scenarios.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6.3
\subsection{Machine Translation\label{sec:mt}}

The evaluation on the generation task, i.e., machine translation (MT), is to assess if the information retrieved by our method is more beneficial for downstream tasks. Instead of training a new MT model \cite{vaswani2017attention}, we propose to integrate the information to the input context of the LLM, e.g., Llama-2 \cite{touvron2023llama}, following the fashion of retrieval-augmented generation (RAG) \cite{lewis-2020-rag}. 



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6.3.1
\subsubsection{Setup} 
We utilize \textsc{Llama-2-7B} \cite{touvron2023llama} model as the backbone model for translation. We use two of our phrase retrieval models for this task, i.e., \textsc{CCPR-Labse} and \textsc{CCPR-XLMR}, as discussed in Section \ref{sec:impl}. We also consider sentence-level retrieval models as baseline methods, i.e., the XLM-RoBERTa-base (\textsc{XLMR}) \cite{conneau-etal-2020-unsupervised} and \textsc{LaBSE} \cite{feng-etal-2022-language}, which are used for retrieving relevant sentences for Llama-2-7B. The two sentence-level baselines are proposed to evaluate whether the phrase retrieval is more beneficial than sentence retrieval for augmenting LLM's cross-lingual ability. The generation quality of each method is evaluated by BERTScore \cite{zhang2019bertscore} and COMET \cite{rei-etal-2020-comet}. 

We integrate the retrieval information into input context of the LLM. An example of the prompt of our method is shown in Figure \ref{tab:ours_prompt}, where the retrieved target phrases with their contexts are presented along with the source phrases in the prompt. The presented contexts are truncated to no more than 100 characters in our method. We use the templates in Figure \ref{tab:llm_prompt} and Figure \ref{tab:sent_prompt} for sentence-retrieval-augmented and vanilla LLM, respectively. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F3
\begin{figure}[b]
\begin{center}
\includegraphics{32-3ia5f03.eps}
\end{center}
\hangcaption{An exemplar instruction of the vanilla LLM. The \textcolor{grey}{text}  after ``English:'' is the reference, which is just for illustration and will not appear in the instruction.}
\label{tab:llm_prompt}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F4
\begin{figure}[t]
\begin{center}
\includegraphics{32-3ia5f04.eps}
\end{center}
\hangcaption{An exemplar instruction of the LLM augmented by cross-lingual sentence retrieval. \textsc{XLMR} and \textsc{LaBSE} are used for retrieval. The \textcolor{grey}{text}  after ``English:'' is the reference, which will not appear in the instruction.}
 \label{tab:sent_prompt}
{\vskip -1pt}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%



Additionally, since the output format of the Llama-base model is hard to control and the Llama-chat model, i.e., the model after instruction-tuning, may reject to translate a sentence that contains negative words, we decide to fine-tune the \textsc{Llama-2-7B} model using LoRA \cite{hu2022lora} for 100 steps to align the translation format. We use Platypus\footnote{\url{https://github.com/arielnlee/Platypus}} \cite{platypus2023} as our code base. We follow all the hyper-parameters of Platypus to fine-tune the LoRA module. The data we used for fine-tuning contain 1,800 parallel sentences randomly sampled from the WMT16 training set. For each method, we only fine-tune two models, i.e., one for the X$\Rightarrow$En direction and the other for En$\Rightarrow$X direction, where X contains all the non-English languages.

The indexing strategies diverge between the baseline methods and our approach. For sentence-level baselines, i.e., \textsc{XLMR} and \textsc{LaBSE}, the indexing is straightforward, directly using the sentences. In contrast, our method employs a learned phrase segmentation module to extract phrases from sentences for indexing. When applying our model to MT, we set the phrase segmentation thresholds in Eq. \ref{eq:pseg} to $0.7$ for indexing and $0.9$ for querying. The rationale behind a lower indexing threshold is to populate the index with a broader array of phrases, enhancing its comprehensiveness. Conversely, a higher threshold for query documents ensures that the segmented phrases are more accurate. These hyper-parameters were tuned using the validation dataset from WMT16. For all methods, we use the monolingual newscrawl data of 2016\footnote{\url{https://data.statmt.org/news-crawl/${LANG}/news.2016.${LANG}.shuffled.deduped.gz}} as the retrieval data.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6.3.2
\subsubsection{Main results} 

As shown in Table \ref{tab:mt}, both of our \textsc{CCPR-XLMR} and \textsc{CCPR-LaBSE} outperform the baseline methods when assisting the LLM with the information from cross-lingual contextualized phrase retrieval. Especially on some low-resource directions, such as the Tr$\Leftrightarrow$En, En$\Rightarrow$Cs, En$\Rightarrow$Fi, \textsc{CCPR-LaBSE} outperforms other baselines by more or around 2 BERTScore points. This indicates that the the contextualized cross-lingual phrase information is more critical for improving the performance on low resource languages, because the percentage of Cs, Fi, and Tr data in Llama-2 training are less than $0.03\%$ \cite{touvron2023llama}. As shown in Table \ref{tab:comet}, the results on COMET is consistent with those on BERTScore.


%%%%%%%%%%%%%%%%%%%%%%%%%T2
\begin{table}[t]
\input{05table02.tex}
\hangcaption{MT on the test sets of WMT16. For all the retrieval-based method, the index is built on the monolingual newscrawl data of the target language. We use the BERTScore as the evaluation metric. The best performance of each translation direction is highlighted in \textbf{bold text}.}
\label{tab:mt}
{\vskip -4pt}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%T3
\begin{table}[t]
\input{05table03.tex}
\hangcaption{MT on the test sets of WMT16. The same setting as Table \ref{tab:mt}. We use the COMET as the evaluation metric. The best performance of each translation direction is in \textbf{bold text}.}
\label{tab:comet}
{\vskip -5pt}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%


In addition, we find a good correlation between the performance of our cross-lingual contextualized phrase retrieval task and the MT task, showing that higher retrieval accuracy leads to superior results in downstream applications.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6.3.3
\subsubsection{Results on east asian languages}
Since most languages in our previous experiments are from Indo-European languages, we further conduct experiments on three East-Asian languages, i.e., Japanese, Korean, and Chinese, to evaluate the effectiveness of our method.


\paragraph{Setup} For each language pair, we use 5 million parallel sentences randomly sampled from the ParaCrawl portion of WMT. We train a single unified phrase encoder for all three language pairs. During translation, we retrieve from monolingual NewsCrawl data (2019). All training and retrieval settings are kept consistent with those used for Indo-European languages.

Following our original setup, the backbone model is LLaMA-7B. Retrieved sentences and phrases are integrated into the input context of the large language model. We evaluate both the baselines and our method using the En$\Rightarrow$Ja, En$\Rightarrow$Ko, and En$\Rightarrow$Zh test sets from NTREX-128 \cite{federmann-etal-2022-ntrex}.

\paragraph{Results} As shown in Table \ref{tab:bertscore_asian} and \ref{tab:comet_asian}, our approach outperform other baselines significantly in terms of the BERTScore and COMET scores. In addition, the translation for all of the East-Asian languages are significant with $p < 0.05$.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T4
\begin{table}[b]
\input{05table04.tex}
\hangcaption{Experiments on NTREX-128 using BERTScore as the evaluation metric. The best results are highlighted by bold text.}
\label{tab:bertscore_asian}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T5
\begin{table}[b]
\input{05table05.tex}
\hangcaption{Experiments on NTREX-128 using COMET as the evaluation metric. The best results are highlighted by bold text.}
\label{tab:comet_asian}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S7
\section{Analysis} 
In this section, we conduct extensive ablation and case studies to evaluate the effectiveness of our proposed CCPR method. The experimental settings used for ablation and case studies are the same as those of the translation task (Section \ref{sec:mt}).



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S7.1
\subsection{Ablation Study}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S7.1.1
\subsubsection{Cross-lingual retrieval} In Table \ref{tab:mt}, all retrieval-based methods construct their index using the monolingual newscrawl data, which is approximately six times the size of the bilingual training data.  It highlights a key benefit of our cross-lingual phrase retrieval: the ability to utilize extensive monolingual resources to build the index \cite{cai-etal-2021-neural}. Therefore, in Table \ref{tab:monolingual}, we evaluate the differences of indexing on monolingual data and the bilingual data. We observe that exploring the vast monolingual data leads to significantly better performance.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T6
\begin{table}[b]
\input{05table06.tex}
\hangcaption{Analysis about the retrieval data for indexing. The ``Train''  and ``NewsCrawl'' indicate using the target sides of WMT16 training data and monolingual newscrawl data to build the index, respectively. We use BERTScore as our evaluation metric.}
\label{tab:monolingual}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S7.1.2
\subsubsection{Phrase segmentation}
In addition, although the $N$-gram is widely used for phrase segmentation in previous works \cite{bapna-firat-2019-non,lee-etal-2021-learning-dense,lee-etal-2021-phrase,lan2023copy}, our experiments demonstrate that a learned phase segmentation is more suitable for augmenting cross-lingual tasks, as shown in Table \ref{tab:tok}. In addition, In Figure \ref{fig:word_stats}, we demonstrate the statistics for the number of words in the retrieved phrases by our method. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T7
\begin{table}[t]
\input{05table07.tex}
\hangcaption{Analysis of phrase segmentation methods. The $N$-gram method uses all available 5-grams for indexing and searching, whereas \textsc{PSM} employs the learned phrase segmentation module. The extracted phrases are encoded by the same encoder, i.e., \textsc{CCPR-LaBSE}. We use BERTScore as the evaluation metric.}
\label{tab:tok}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F5
\begin{figure}[t]
\begin{center}
\includegraphics{32-3ia5f06.eps}
\end{center}
    \caption{Statistics for the number of words in retrieved phrases.} 
    \label{fig:word_stats}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S7.1.3
\subsubsection{Retrieval augmented generation}
To understand where the performance gain comes from, we also compare the CCPR with the the Chain-of-Though (CoT) approach \cite{wei2022chain}. This comparison aims to understand whether  the phrase translation can be induced from the LLM without retrieving on external data-store. The prompt we used for the CoT approach is shown in Figure \ref{fig:cot_prompt}. As shown in Table \ref{tab:cot_result}, simply translating phrases before sentence translation does not achieve the expected performance gain and is slightly worse than the baseline approach. In contrast, our approach outperforms this method by a large margin.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F6
\begin{figure}[t]
\begin{center}
\includegraphics{32-3ia5f06.eps}
\end{center}
\hangcaption{An exemplar template for the translation with Chain-of-Thought. The \textcolor{grey}{text} is the reference, which is just for illustration and will not appear in the input.}
 \label{fig:cot_prompt}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T8
\begin{table}[t]
\input{05table08.tex}
\hangcaption{Translation results based on \textsc{Llama-7-7B}. For the CoT method, we use the template in \mbox{Figure \ref{fig:cot_prompt}, s}and all the rest setups are the same as other methods in our work. We use the BERTScore as the metric}
 \label{tab:cot_result}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%





%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S7.2
\subsection{Case Study}
In Figure \ref{tab:case_study}, we compare examples translated by our method, i.e., \textsc{CCPR-LaBSE}, with those produced by the vanilla and sentence-retrieval-augmented LLMs, i.e., \textsc{Llama-2-7B} and \textsc{LaBSE}. Examples from Turkish and German to English demonstrate that our method not only provides helpful phrase-level information to the LLM but also increases the coverage of the translation.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F7
\begin{figure}[t]
\begin{center}
\includegraphics{32-3ia5f07.eps}
\end{center}
\hangcaption{Example of handling the polysemy using our method. The \textcolor{grey}{text in grey} is provided to assist with the understanding of the Chinese sentences.}
 \label{tab:case_study}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


For instance, in the Tr$\Rightarrow$En task, our approach correctly translates ``AB b{\={u}}t{\k{c}}esinden'' to ``EU budget'' and ``milyar Euro'' to ``billion euros'' thanks to cross-lingual phrase retrieval. However, both \textsc{Llama-2-7B} and \textsc{LaBSE} fail to capture the semantics of ``AB b{\={u}}t{\k{c}}esinden'' and ``milyar Euro''. Although the sentence-retrieval-augmented LLM also leverages an external datastore, it cannot retrieve fine-grained information as effectively as our approach. Additionally, since CCPR attempts to retrieve all potential phrases in the source sentence, it ensures that the LLM pays attention to all parts of the source sentence. This implicitly improves the translation coverage of the system.

{
Moreover, an example demonstrating how our method leverages contextual information to handle polysemy is shown in Figure \ref{fig:polysemy}. The baseline method mistranslated ``muppet'' as``moppet'', where the former means ``a stupid person'' and the latter refers to ``a baby or child''. In contrast, our method utilizes contextual information to retrieve a cross-lingual sentence with highly similar semantics and structure. In this sentence, ``小丑'' (clowns) conveys a meaning closer to ``muppets''. As a result, our approach achieves a translation with improved accuracy and better evaluation scores.
}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F8
\begin{figure}[t]
\begin{center}
\includegraphics{32-3ia5f08.eps}
\end{center}
\hangcaption{Case Study. The \underline{phrases with underline} in \textbf{SRC} and \textbf{HYP} are aligned phrases extracted and retrieved by our CCPR method, respectively.}
\label{fig:polysemy}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%





%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S8
\section{Conclusion}
This paper introduces a novel approach to dense retrieval, i.e., cross-lingual contextualized phrase retrieval, focusing on resolving phrase polysemy by utilizing contextual information. The main challenge identified is the scarcity of training data, specifically cross-lingual phrase pairs with context. To overcome this, we use a word alignment model to derive such phrase pairs from parallel sentences. We then present the Cross-lingual Contextualized Phrase Retriever (CCPR), which employs contrastive learning to effectively capture similar meanings and contexts of cross-lingual phrases. Our extensive testing across retrieval and machine translation tasks shows that CCPR significantly outperforms existing baselines.



% Start bibliography
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.7}
\bibliography{05refs}


% Start biography
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Huayang Li}{%
Ph.D. student at the Nara Institute of Science and Technology starting in 2023. Huayang received the Master's degree from NAIST in 2023. Before joining NAIST, he worked at the Natural Language Processing Center of Tencent AI Lab as a researcher between 2018 and 2021. He got the bachelor degree from Central China Normal University in 2018.
}

\bioauthor[:]{Deng Cai}{He is an independent research scientist in China. He has a broad research interest in natural language processing and machine learning. His work won an Outstanding Paper Award at ACL (2021) and a Best Paper Nomination at ACMMM (2024). He delivered cutting-edge tutorials at IJCAI (2022), SIGIR (2022), and CCL (2022). He served as area chairs for top-tier AI conferences such as NeurIPS, ACL, EMNLP, and NAACL. He was selected for the Young Elite Scientist Sponsorship Program by the China Association for Science and Technology. Website: \url{https://jcyk.github.io/}}

\bioauthor[:]{Zhi Qu}{Ph.D. student at the Nara Institute of Science and Technology (NAIST), starting in 2023. He received his Master's degree from NAIST in 2023 and his Bachelor's degree from Chongqing Normal University, China, in 2018. Currently, he is on the internship at Advanced Translation Technology Laboratory of National Institute of Information and Communications Technology (NICT), Japan.}


\bioauthor[:]{Qu Cui}{He received the M.S. degree in Computer Science from Nanjing University in 2021, where he worked at the National Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China. He earned his B.S. degree in Software Engineering from Northwestern Polytechnical University in 2018. Qu Cui is currently employed as a Research Engineer at Tencent.}

\bioauthor[:]{Hidetaka Kamigaito}{He received Ph.D. from Tokyo Institute of Technology. After being an assistant professor at Tokyo Institute of Technology, he is currently an associate professor at Nara Institute of Science and Technology. His current research interests include natural language processing with an emphasis on natural language generation and knowledge graph completion.}

\bioauthor[:]{Lemao Liu}{He received the Ph.D. degree from the Harbin Institute of Technology, Harbin, China. He is currently a Principal Researcher with the Natural Language Processing Center, Tencent AI Lab, Shenzhen, China. Previously, he was with the National Institute of Information and Communications Technology, Japan. He has authored or coauthored about 50 research papers in leading conferences and journals, such as ACL, EMNLP, NAACL, COLING, ICLR, AAAI, and JAIR. His research interests include machine translation, syntactic parsing, and natural language understanding. He was the recipient of the outstanding paper award in ACL 2021 and the best demo award in CCL 2020. He was a publication co-chair of EMNLP 2020 (Findings), and a senior program committee Member of IJCAI 2021.}

\bioauthor[:]{Taro Watanabe}{%
He received his B.E. and M.E. degrees in information science from Kyoto University in 1994 and 1997, respectively, and obtained an M.S. degree in Language and Information Technologies from the School of Computer Science, Carnegie Mellon University in 2000.
In 2004, he received a Ph.D. in informatics from Kyoto University.
After working as a researcher at ATR, NTT and NICT, and as a software engineer at Google, he is a professor at the Nara Institute of Science and Technology starting in 2020.
His research interests include natural language processing, machine learning, language modeling and machine translation.
}
\end{biography}

\biodate   %%%% Following settings will be edited by editors.


\end{document}

