\documentclass[english]{jnlp_3.6}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{hangcaption_jnlp}
%%%%\usepackage{udline}
%%%%\setulminsep{1.2ex}{0.2ex}
%%%%\let\underline=\ul
\allowdisplaybreaks


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
%%%%\usepackage{amssymb}
\usepackage{csquotes}
\usepackage{multirow}
\usepackage{booktabs}
%%%%\usepackage{algorithm}
%%%%\usepackage{algpseudocode}
%%%%\usepackage{url}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{30}
\Number{2}
\Month{June}
\Year{2023}
%%%%\Issuetitle{Special Issue: XXXXXXXX}

\received{2022}{9}{1}
\revised{2022}{12}{1}
\accepted{2023}{1}{10}

\setcounter{page}{304}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Enhancing Semantic Correlation between Instances and Relations for Zero-Shot Relation Extraction}
\eauthor{Van-Hien Tran\affiref{Author_1} \and Hiroki Ouchi\affiref{Author_2} \and Hiroyuki Shindo\affiref{Author_2} \and Yuji Matsumoto\affiref{Author_3} \and Taro Watanabe\affiref{Author_2}} 
\eabstract{%
Zero-shot relation extraction aims to recognize (new) unseen relations that cannot be observed during training. Due to this point, recognizing unseen relations with no corresponding labeled training instances is a challenging task. Recognizing an unseen relation between two entities in an input instance at the testing time, a model needs to grasp the semantic relationship between the instance and all unseen relations to make a prediction. This study argues that enhancing the semantic correlation between instances and relations is key to effectively solving the zero-shot relation extraction task. A new model entirely devoted to this goal through three main aspects was proposed: learning effective relation representation, designing purposeful mini-batches, and binding two-way semantic consistency. Experimental results on two benchmark datasets demonstrate that our approach significantly improves task performance and achieves state-of-the-art results. Our source code and data are publicly available.}
\ekeywords{Information Extraction, Zero-Shot Learning, Relation Extraction, Semantic Correlation}

\headauthor{Tran et al.}
\headtitle{Enhancing Semantic Correlation between Instances and Relations for ZSRE}

\affilabel{Author_1}{}{National Institute of Information and Communications Technology (NICT)}
\affilabel{Author_2}{}{Nara Institute of Science and Technology, Japan}
\affilabel{Author_3}{}{RIKEN Center for Advanced Intelligence Project (AIP)}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}
\label{sec:introduction}

Relation extraction is a fundamental task in natural language processing (NLP), which identifies the semantic relation between two entities in an input instance.\footnote{As \enquote{input instance} and \enquote{input sentence} are interchangeable, they are referred to as \enquote{input instance}.}
It has attracted much research effort because it plays a vital role in many NLP applications, such as information extraction \cite{tran2021improved,tran-etal-2021-covrelex}, knowledge graph construction \cite{lin2015learning,vu2015method} and question answering \cite{xu2016question}.
This task is generally a supervised classification problem by training models on large-scale labeled data.
Following this approach, existing supervised models \cite{tran-etal-2019-relation,pouran2020exploiting,tian2021dependency} have achieved an excellent performance.


However, when putting relation extraction in real-world scenarios where many \textit{new relations} always exist, the current supervised models cannot recognize new relations well because they are unobserved during training.
Therefore, it is worth inventing models capable of identifying \textit{new relations} that have never been seen before.
This task is called zero-shot relation extraction (ZSRE), where a model is trained on labeled instances of the seen relations but then targeted to predict unseen relations for testing instances.
Additionally, information on all unseen relations is at the testing stage, including their labels (required information) and descriptions (optional information).
Thus, to make a correct prediction, the model must profoundly understand the semantic relationship between each instance and all unseen relations.
Following this intuitive reasoning, we argue that enhancing the semantic correlation between instances and relations is the key to solving ZSRE effectively.


While relevant studies on ZSRE are still limited, these studies underestimated the key solution above and had some other limitations.
For example, \citeA{levyetal2017zero} formulated ZSRE as a question-answering task by creating manually predefined question templates for each relation.
However, it is infeasible and impractical to make such human efforts for many new unseen relations in the zero-shot setting.
\citeA{obamuyidevlachos2018zero} reduced ZSRE to a text entailment task and designed a binary classifier to indicate whether a given relation description depicts the relationship between two entities in an input instance.
This approach requires the inefficient execution of multiple binary classifications over all relation descriptions and cannot make relations comparable.


More recently, \citeA{chenli2021zs} presented a model called ZS-BERT, which first projects instances and relations in a shared space and then minimizes the distance between each instance and the corresponding relation.
However, although ZS-BERT considers the semantic relationship between instances and relations, it has a severe limitation.
Specifically, as relation representations are fixed during training, it leads to low-quality relation representations and hinders grasping the semantic relationship.
\citeA{gong2021prompt} then introduced a prompt-based model with semantic knowledge augmentation to recognize unseen relations.
They initially generated augmented instances with unseen relations from training instances with seen relations.
Then, they designed prompts based on an external knowledge graph to learn representations for both seen and unseen relations.
However, this model requires unseen relation labels and an external knowledge graph in the training stage, although such information is not readily available in real-world scenarios.
\citeA{tran-etal-2022-improving} proposed a new model which incorporates discriminative embedding learning for both sentences and semantic relations and uses a self-adaptive comparator network to judge whether the relationship between a sentence and a relation is consistent. Although this method helps boost the performance task significantly, using only a simple comparator network, a two-layer nonlinear neural network, may need better for grasping the semantic consistency between sentences and relations.



While there are few research works on the ZSRE task, zero-shot learning (ZSL) on other tasks, like recognizing unseen objects, has attracted much research effort. Specifically, it links the seen and unseen classes through the semantic information to complete the object recognition task. For bridging visual space and semantic space, most traditional ZSL methods tend to learn a mapping that projects samples from the visual space to the semantic space with the labeled training set, including seen classes only. For classifying unseen images, the learned embedding is used to project the visual representation of unseen samples into the semantic space, including unseen and seen classes. Then, the nearest neighbor search method is used to recognize the sample of unseen class in the testing phase. \citeA{sung2018learning} proposed a model named Relation Network (RN) to measure the relation score between unseen samples and semantics by
learning a distance metric. \citeA{zhang2019co} presented a Co-Representative network (CRnet) based on RN to learn a uniform embedding space by a single layer module with parallel structure and high local. However, CRnet relies heavily on obtaining human-defined semantics for knowledge transfer. Besides, a few works \cite{yu2019hierarchical,hou2020discriminative} have been proposed to maintain the relationship between semantics by using complementary features or extracting deep local features, although these features are rarely used in the ZSL task. \citeA{liu2021relation} proposed
a relation-based discriminative cooperation network (RDCN) model for ZSL to overcome the domain bias problem in most previous methods. \citeA{yun2022attributes} presented a Salient Attributes Learning Network (SALN) to generate discriminative and expressive semantic representation under the supervision of the visual features. 


In this paper, we propose a new approach that is purposely designed to solve the ZSRE task by overcoming the limitations of the previous studies.
Without any external knowledge graphs or unseen relation labels in the training phase, our model focuses on effectively grasping the semantic correlation between instances and relations because it is a crucial solution for solving the ZSRE.
Our model achieves this by concentrating on the following three aspects.



First, our model acquires meaningful and high-quality representations for instances and relations.
This aspect plays an essential role in understanding the semantic correlation between instances and relations. 
Specifically, instead of using fixed pre-trained relation representations, as in the previous work \cite{chenli2021zs},
our approach obtains the instance and relation representations via a learnable BERT-based encoder module.
We also exploit relation labels and relation descriptions to attain the better relation representations.


Second, the previous studies \cite{chenli2021zs,gong2021prompt,tran-etal-2022-improving} prepared mini-batches in a standard manner, where each training mini-batch comprises some of the labeled instances by a random sampling technique.
In contrast to this approach, we design each mini-batch as a mini-task, including $K$ different seen relations and $K$ corresponding instances ($K$ is a hyperparameter), and force the model to pair them exactly.
This strategy encourages the model to deeply grasp the semantic relationship between instances and relations.


Finally, the previous studies \cite{chenli2021zs,gong2021prompt} treat relation representations as targets and minimize the probability distribution from each instance to its corresponding relation in the shared space.
This approach is a one-way interaction that cannot fully exploit the semantic relationship between instances and relations.
Instead, we use two-way interaction, which grasps the interaction not only \enquote{from each instance to relations} but also \enquote{from each relation to instances} and constrains the consistency of the two interaction distributions.

The contributions of this study are summarized as follows:

(a) We indicate that enhancing the semantic correlation between instances and relations is key to drastically improving the performance of ZSRE. %successfully solving ZSRE. 

(b) We propose an approach that focuses on this goal by learning high-quality relation representation, designing strategic mini-batches, and binding two-way semantic consistency.

(c) Extensive experiments on two benchmark datasets demonstrated the effectiveness and robustness of our approach, as it significantly outperformed the existing state-of-the-art methods.


It can be seen that our proposed model is closely related to dense retrieval models. Specifically, dense retrieval models aim to retrieve relevant documents for a given query in information retrieval research applications. They try to capture the deep semantic relationship between queries and documents in embedding space by mapping documents and queries to k-dimensional real-valued vectors. Existing dense retrieval models can be classified into two categories. One line of research is negative sampling \cite{karpukhin-etal-2020-dense,xiong2021approximate,Zhan2021OptimizingDR}, while the other line is knowledge distillation \cite{qu2020rocketqa,lin2020distilling,hofstatter2021efficiently}, which adopts a cross-encoder to generate pseudo labels. The negative sampling approach selects several negative documents from the entire corpus for a given training query. Then, the dense retrieval model encodes the queries and documents into embeddings and uses the inner product to compute their relevance scores. The training method uses the scores to compute a pairwise loss based on the gold annotations. Our proposed model for the ZSRE task is close to this method but still different. Concretely, we design each purposeful mini-batch including $K$ relations and $K$ corresponding instance and force the model to pair them exactly. Besides, we also propose to put the added constraint using the KL-Divergence Loss to improve the semantic relationship between instances and relations, which has never been suggested before.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Work}

Only a few relevant studies have been conducted on ZSRE. \citeA{levyetal2017zero} regarded ZSRE as a question-answering task.
They first manually defined $10$ question templates to represent each relation type and then made predictions by training a reading comprehension model to determine which relation satisfies the given instance and question.
Because this method requires human effort to define question templates for unseen relations, it is possibly infeasible and impractical to prepare such templates for multiple new unseen relations in real-world scenarios.
\citeA{obamuyidevlachos2018zero} formulated ZSRE as a textual entailment task, where the input instance with two entities is the premise \textit{P}, whereas the relation description is the hypothesis \textit{H}.
They then used existing textual entailment models and required the models to predict whether \textit{P} matches \textit{H}.
Specifically, they adopted the enhanced sequential inference model (ESIM) \cite{chen2017enhanced} and conditioned inference model (CIM) \cite{rocktaschel2015reasoning} as their base models.


Recently, \citeA{chenli2021zs} presented a model called ZS-BERT, which learns two functions to project sentences and relation descriptions into an embedding space by jointly minimizing the distances between them and classifying the seen relations. 
ZS-BERT then uses the nearest neighbor search to obtain the prediction of unseen relations, although this technique is prone to suffering from the hubness problem \cite{radovanovic2010}.
Another severe problem is relation representations generated by feeding their relation descriptions into the frozen pre-trained Sentence-BERT \cite{gurevych2019sentence}. 
These relation representations were fixed during the training. 
This hinders the learning of meaningful relation representations, thereby affecting task performance.


\citeA{gong2021prompt} proposed a prompt-based model with semantic knowledge augmentation (ZS-SKA) to recognize unseen relations under the zero-shot setting.
They generated augmented instances with unseen relations from instances with seen relations following a new word-level sentence translation rule.
By creating representations of the seen and unseen relations with augmented instances and prompts through prototypical networks, the distance between each query instance and all prototype embeddings of all relations are calculated for prediction.
This approach requires the provision of unseen relation labels and external knowledge graphs during the training phase.
Thus, it is impractical and infeasible in real-world scenarios because the required information is not readily available at the training stage. 



While related works on the ZSRE task are limited, ZSL on other tasks, like recognizing unseen objects, has attracted much research. Related works on the object recognition task can be divided into two approaches: relation-based models and synthesis-based models. For the \textbf{first approach}, relation-based models aim to learn the relationship between images and the semantics by using a Visual-Semantic Embedding (VSE) framework with three different directions of mappings: (1) Visual \textit{to} Semantic Embedding, (2) Semantic \textit{to} Visual Embedding, and (3) Visual and Semantic \textit{to} Common Space. The first direction \enquote{Visual \textit{to} Semantic Embedding} aims to learn a mapping from
the visual space to the semantic space either using linear function \cite{kodirov2017semantic,shi2020discriminative} or by deep neural network regression \cite{annadani2018preserving}. Meanwhile, the second direction \enquote{Semantic \textit{to} Visual Embedding} tries to learn a mapping from the semantic space to the visual space \cite{zhang2020pseudo,liu2020simple,zhang2020deep}. Different from the two directions above, the last one \enquote{Visual and Semantic \textit{to} Common Space} learns a common space where both the visual features and the semantics are embedded together, such as \cite{wang2017zero,xing2020zero}.
For the \textbf{second approach}, synthesis-based models \cite{gao2020zero,gull2022generalized} make an effort first to learn a generator that generates samples from the semantics and then train a classifier to predict classes with the generated synthesis samples. For instance, \citeA{xian2018feature} introduced f-CLSWGAN that creates sufficiently discriminative CNN features to train softmax classifiers with the help of a Wasserstein Generative Adversarial Networks. \citeA{luo2021dual} presented a dual framework based on variational auto-encoder (VAE) and generative adversarial network (GAN), known as dual VAEGAN, to produce more clear visual features than VAE and alleviate the model collapse problem of GAN. More recently, \citeA{gull2022generalized} proposed an identifiable VAE (iVAE) based generative model for learning disentangled representations, where each dimension is statistically independent and responsible for generating data. The advantage of synthesis-based models is that both seen and unseen samples may be obtained by the generator, which significantly alleviates the domain bias problem. 



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Task Definition}

We follow the exact definition of ZSRE from previous works \cite{chenli2021zs,gong2021prompt} to introduce the task.

Let $\mathcal{Y}_{\mathcal{S}}=\left\{y_{s}^{1}, \ldots, y_{s}^{n}\right\}$ and $\mathcal{Y}_{\mathcal{U}}=\left\{y_{u}^{1}, \ldots, y_{u}^{m}\right\}$ denote the sets of \textit{seen} and \textit{unseen} relation labels, respectively, where $n=\left|\mathcal{Y}_{\mathcal{S}}\right|$ and $m=\left|\mathcal{Y}_{\mathcal{U}}\right|$ denote the numbers of relations between the two sets.
These two sets are disjoint, \textit{i.e.}, $\mathcal{Y}_{\mathcal{S}} \cap \mathcal{Y}_{\mathcal{U}}=\emptyset$. Given a training set with $N$ samples, the $i^{th}$ sample comprises the input instance $X_{i}$, the entities $e_{i1}$ and $e_{i2}$, and description $D_{i}$ of the corresponding \textit{seen} relation label $y_{s}^{i} \in \mathcal{Y}_{\mathcal{S}}$, hereby denoted as $\left\{S_{i}=\left(X_{i}, e_{i 1}, e_{i 2}, D_{i}, y_{s}^{i}\right)\right\}_{i=1}^{N}$.
Note that, while relation label information is compulsory, relation description information is optional according to its availability.
Using the training set, our goal is to train a model $\mathcal{M}$, \textit{i.e.}, $\mathcal{M}\left(S_{i}\right) \rightarrow y_{s}^{i} \in \mathcal{Y}_{\mathcal{S}}$.
In the testing stage, given a testing instance $S^{\prime}=\left(X, e_{1}, e_{2}\right)$ comprising the input instance $X$ with two provided entities, and all \textit{unseen} relation labels in $\mathcal{Y}_{\mathcal{U}}$ (required information) and their descriptions (optional information), $\mathcal{M}$ predicts the \textit{unseen} relation label $y_{u}^{j} \in \mathcal{Y}_{\mathcal{U}}$ for $S^{\prime}$.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Approach}
\label{main_sec_approach}

This section presents the details of the proposed approach for solving the ZSRE.
Figure~\ref{our_model} shows the overall learning framework and illustrates an example of the prepared mini-batch for training the framework.
This model aims to enhance the semantic correlation between instances and relations.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1
\begin{figure}[b]
\begin{center}
\includegraphics{30-2ia2f1.pdf}
\end{center}
\hangcaption{The overall framework of our approach. The input is a training mini-batch that consists of $K$ different relations (e.g., $K=3$) and $K$ corresponding instances. The circles and rectangles represent relations and instances, respectively. The circle and the rectangle with the same color is a pair. Distinct colors represent different classes. For simplicity, we illustrate interactions from only one instance to relations and only one relation to instances on the right side. As instance-to-relations classification is the original ZSRE task, then $\mathcal{L}_{CE}$ is used to supervise it. Further, we use $\mathcal{L}_{KL}$ to put an added constraint on the correlation between the two distributions.}
\label{our_model}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Instance Representation}

We use BERT \cite{devlin2019bert} as the basic encoder to generate contextualized representations of input instances.
Following \citeA{baldinisoares2019matching}, we first augment each input instance with four reserved word pieces ([E1], [/E1], [E2], and [/E2]) to indicate two entities in the input instance.
For example, the input instance is   
\enquote{\emph{In 1959, along with his family, [Gene Chen]\textsubscript{\textit{e1}} moved to the USA and settled in [San Francisco]\textsubscript{\textit{e2}}.}} becomes  \enquote{\emph{In 1959, along with his family, [E1] Gene Chen [/E1] moved to the USA and settled in [E2] San Francisco [/E2] .}}.
We then feed this sequence into the BERT encoder.
Finally, the input instance representation is obtained by concatenating two hidden state vectors of the two start tokens ([E1] and [E2]) with the final dimension of $\mathbb{R}^{2d}$. In Figure~\ref{our_model}, an instance representation is illustrated by a colored rectangle.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Relation Representation}
\label{sec_rel_rep}

A BERT-based encoder is also used to obtain semantic relation representations.
As introduced in the ZSRE task definition, while relation label information is compulsory, relation description is optional according to its availability. 
Thus, if only the relation label is provided, we input it into the BERT encoder.
Conversely, when both relation labels and descriptions are given, we initially concatenate them using a special token: [SEP].
For example, the relation label \enquote{\textit{residence}} with its description:\footnote{\url{https://www.wikidata.org/wiki/Property:P551}} \enquote{\textit{the place where the person is or has been, resident}} becomes \enquote{\textit{residence [SEP] the place where the person is or has been, resident}}.
Then, we feed it into the BERT encoder.


In both cases, the final relation representation is attained by the hidden states corresponding to the [CLS] token (converted to $\mathbb{R}^{2d}$ dimension with a linear transformation).
The linear transformation guarantees equal dimension sizes of the instance and relation representations.
In Figure~\ref{our_model}, a relation representation is illustrated by a colored circle.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Semantic Correlation Learning}
\label{sec_semcor}

Given a training set with $N$ samples of $T$ different \textit{seen} relations ($T<N$), we create mini-batches to train our model. To facilitate the model in grasping the semantic correlation between instances and relations, we intentionally design mini-batches differently. Each mini-batch consists of $K$ different \textit{seen} relations ($K\leq T$), randomly sampled, and $K$ corresponding instances. The model is then required to match instances to the corresponding relations exactly.


We feed each mini-batch into the BERT-based encoder and obtain $K$ relation representations: $\left\{\mathbf{r}_{i} \in \mathbb{R}^{2 d} ; i=1, \ldots, K\right\}$ and $K$ instance representations: $\left\{\mathbf{s}_{i} \in \mathbb{R}^{2 d} ; i=1, \ldots, K\right\}$.
Note that the $i^{th}$ relation has a corresponding $i^{th}$ instance.
We encourage mutual interaction between instances and relations to help the model grasp the semantic correlation in depth.
Specifically, in Figure~\ref{our_model}, after acquiring representations of the instances and relations, we consider each $i^{th}$ pair ($\mathbf{s}_{i}$, $\mathbf{r}_{i}$) in turn.
From the instance $\mathbf{s}_{i}$, we first compute its similarity to all $K$ relations using the dot product operation and then use softmax to obtain a probability distribution over 
\pagebreak
the $K$ relations as follows:
\begin{equation}
z_{ij}=\frac{\exp \left(\mathbf{s}_{i} \cdot \mathbf{r}_{j}\right)}{\sum_{k=1}^{K} \exp \left(\mathbf{s}_{i} \cdot \mathbf{r}_{k}\right)}
\end{equation}
where $z_{ij}$ is the estimated probability for the $j^{th}$ relation of the $i^{th}$ instance.
Let $\mathbf{z}_{i} = $\linebreak$
[ z_{i1}, z_{i2}, \ldots, z_{iK} ]$ denote the probability distribution of the $i^{th}$ instance, which sums up to $1$.
The cross-entropy loss is calculated as follows:          
\begin{equation}
\mathcal{L}_{CE}=-\log \left(z_{ii}\right),
\end{equation}
where the $i^{th}$ instance has a corresponding ground-truth $i^{th}$ relation in the $i^{th}$ pair.


Similarly, we consider the interaction from each relation to instances.
In Figure~\ref{our_model}, from the relation $\mathbf{r}_{i}$ of the $i^{th}$ pair ($\mathbf{s}_{i}$, $\mathbf{r}_{i}$), we compute its similarity to all $K$ instances using the dot product operation and then use softmax to obtain a probability distribution over the $K$ instances as follows:
\begin{equation}
u_{ij}=\frac{\exp \left(\mathbf{r}_{i} \cdot \mathbf{s}_{j}\right)}{\sum_{k=1}^{K} \exp \left(\mathbf{r}_{i} \cdot \mathbf{s}_{k}\right)}
\end{equation}
where $u_{ij}$ is the estimated probability for the $j^{th}$ instance of the $i^{th}$ relation. Let  $\mathbf{u}_{i} = $\linebreak$
[ u_{i1}, u_{i2}, \ldots, u_{iK} ]$ denote the probability distribution, which sums up to $1$.


By considering the two-way interaction between instances and relations, for each $i^{th}$ pair ($\mathbf{s}_{i}$, $\mathbf{r}_{i}$), we obtain two corresponding probability distributions ($\mathbf{z}_{i}$ and $\mathbf{u}_{i}$).
These two distributions should be consistent to encourage the semantic correlation between the instance $\mathbf{s}_{i}$ and the relation $\mathbf{r}_{i}$.
Since the relationship between instances and relations is symmetric, if an instance is similar to a relation, then the relation will also be similar to the instance, and in verse.
Thus, constraining the consistency between $\mathbf{z}_{i}$ and $\mathbf{u}_{i}$ will strengthen the semantic correlation between instances and relations.
We then use the Kullback-Leibler (KL) divergence loss to supervise this consistency.
Here, we deliberately use $D_{KL}\left(\mathbf{u}_{i} \| \mathbf{z}_{i}\right)$, instead of $D_{KL}\left(\mathbf{z}_{i} \| \mathbf{u}_{i}\right)$ or Jensen-Shannon divergence $D_{JS}\left(\mathbf{u}_{i} \| \mathbf{z}_{i}\right)$.\footnote{We tested with $D_{KL}\left(\mathbf{u}_{i} \| \mathbf{z}_{i}\right)$, $D_{KL}\left(\mathbf{z}_{i} \| \mathbf{u}_{i}\right)$, and $D_{JS}\left(\mathbf{u}_{i} \| \mathbf{z}_{i}\right)$ in our model. Using the loss $D_{KL}\left(\mathbf{u}_{i} \| \mathbf{z}_{i}\right)$ gave the best performance.}


Because the natural language is highly flexible, a relation can be expressed using different textual patterns surrounding two entities in instances.
For example, the relation \enquote{\textit{per:employee\_of}} can be reflected via patterns such as \enquote{worked for}, \enquote{founded and headed}, and \enquote{the CEO of}.
Thus, using the loss $D_{KL}\left(\mathbf{u}_{i} \| \mathbf{z}_{i}\right)$, which promotes $\mathbf{u}_{i}$ to be similar to $\mathbf{z}_{i}$, constrains the consistency of the two distributions and further encourages the model to learn richer 
\pagebreak
and more diverse relation representations according to instances. The loss $D_{KL}\left(\mathbf{u}_{i} \| \mathbf{z}_{i}\right)$ is formulated:
\begin{equation}
\mathcal{L}_{KL}=D_{KL}\left(\mathbf{u}_{i} \| \mathbf{z}_{i}\right)=-\sum_{k=1}^{K} u_{ik} \log \frac{z_{ik}}{u_{ik}}
\end{equation}


The final objective function of the model is defined as follows:
\begin{equation}\label{eq_final_loss}
\mathcal{L}=\mathcal{L}_{CE}+\alpha \cdot \mathcal{L}_{KL}
\end{equation}
where $\alpha$ is a hyperparameter that balances these two terms.
Note that, for each training mini-batch that includes $K$ different relations and $K$ corresponding instances, we accumulate the losses of all these $K$ pairs following the above formulation before using back-propagation in training.




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experiments}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Experimental Setup}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Datasets}

Following previous studies \cite{chenli2021zs,gong2021prompt}, we evaluate our model on two benchmark datasets: \textbf{FewRel} \cite{han2018fewrel} and \textbf{Wiki-ZSL} \cite{chenli2021zs}.
FewRel is a human-annotated balanced dataset comprising $80$ relation types, each with $700$ instances.
Although FewRel was initially used for a few-shot learning task, it is also relevant for zero-shot learning, if the relation labels within training and test sets are disjoint. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[b]
\vspace{-1\Cvs}
\input{02table01.tex}
\caption{Statistics of the datasets. \enquote{avg. len.} stands for the average instance length.}
\label{tab_statistics}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

In contrast, Wiki-ZSL originated from Wiki-KB \cite{sorokin2017context} and is generated with distant supervision.
From the Wiki-KB dataset, \citeA{chenli2021zs} neglected instances with the relation \enquote{none}.
To ensure sufficient data instances for each relation in zero-shot learning, they filtered out relations that appeared below $300$ times.
Finally, they obtained Wiki-ZSL, a subset of Wiki-KB.
The statistics for Wiki-KB, Wiki-ZSL, and FewRel are shown in Table~\ref{tab_statistics}.
Note that descriptions of all relations in Wiki-ZSL and FewRel are available from open-source Wikidata.\footnote{\url{https://www.wikidata.org/wiki/Wikidata:Main_Page}}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Zero-shot Settings}

We follow the experimental settings of \citeA{chenli2021zs} to enable the zero-shot relation extraction scenario.
We randomly select $m$ relations as \textit{unseen} ones ($m=\left|\mathcal{Y}_{u}\right|$), thereby splitting the entire dataset into training and test sets; here, the test set includes all instances belonging to these $m$ relations and the training set with all remaining instances.
This ensures that these $m$ relations are not in the training data such that $\mathcal{Y}_{\mathcal{S}} \cap \mathcal{Y}_{\mathcal{U}}=\emptyset$.
Note that we repeat the experiment $5$ times for $5$ different random selections of $m$ and report the average results.
The evaluation metrics macro precision (\textit{P}), macro recall (\textit{R}), and macro F1-score (\textit{F1}) are also used in this study, similar to previous studies.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Implementation Details}

Our approach is implemented using PyTorch \cite{paszke2019pytorch} and all experiments are performed on $1$ NVIDIA RTX A6000 GPU.
We adopt the transformer library of Huggingface \cite{wolf2020transformers} and use the uncased model of BERT$_{\textit{base}}$ as the encoder.
The AdamW optimizer \cite{Loshchilov2019DecoupledWD} is applied to minimize loss.
For the hyperparameters, $\alpha$ is set to $1$, and $K$ is set to $5$ on FewRel and Wiki-ZSL datasets.
The maximum length of the instances is set to $128$.
The initial learning rate is $2e-5$.
The number of sampled mini-batches is $40,000$.
The hidden size, $d$ is $768$.
The average runtime of our model's training and evaluation is $1.8$ hours on Wiki-ZSL, whereas this number is $4.5$ hours on FewRel. We release our source code\footnote{\url{https://github.com/vhientran/Code-ZSRE}} and data publicly to support future research.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Results and Analysis}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Comparison to Baselines}

The proposed approach is compared with the following baseline methods: \textbf{CNN} \cite{zeng2014relation}, \textbf{Bi-LSTM} \cite{zhang2015bidirectional}, \textbf{Attention-based Bi-LSTM} \cite{zhou2016attention}, \textbf{R-BERT} \cite{wu2019enriching}, \textbf{ESIM} \cite{chen2017enhanced}, \textbf{CIM} \cite{rocktaschel2015reasoning}, and \textbf{ZS-BERT} \cite{chenli2021zs}.
These baselines were reported by \citeA{chenli2021zs}.
We further compare our model with the most state-of-the-art model, \textbf{ZS-SKA}\footnote{As the dataset and source code of ZS-SKA are not available, it is hard for us to reproduce this work.} by \citeA{gong2021prompt}.


The experimental results obtained by varying \textit{$m$} unseen relations (i.e., $m=\{5, 10, 15\}$) are shown in Table \ref{main_result}. We observe that the task performance of all models decreases when $m$ is larger.
There are two possible reasons for this phenomenon.
First, following the experiment settings of the ZSRE, since the whole dataset of $N$ relations is divided into train set (($N-m$) seen relations) and testing set ($m$ unseen relations), the number of seen relations for the training phase will be smaller when \textit{$m$} is larger. 
Second, an increase in $m$ also leads to a rise in the possible choices for prediction, thereby making it more difficult to predict the correct unseen relation.
Thus, $m=15$ is the most complex case in the three cases above to challenge models to achieve a good performance.
Considering this aspect, \citeA{gong2021prompt} only tested their model ZS-SKA on $m=15$ and did not report results on the two remaining cases of $m=5$ and $m=10$. Meanwhile, \citeA{chenli2021zs} evaluated their model ZS-BERT more comprehensively on all three cases of $m$.
We also thoroughly assess our model on all three different values of $m$.



As shown in Table~\ref{main_result}, our model steadily outperforms the competing methods on the Wiki-ZSL and FewRel datasets when considering all different values of $m$.
We specially compare our model to the recent most robust baseline models, including ZS-BERT and ZS-SKA.
Compared with ZS-BERT in all three cases of various $m$ values, our approach significantly outperforms ZS-BERT by a significant margin (at least $20.58$ points in F1-score), particularly for Wiki-ZSL.
Interestingly, our model achieves the best improvement in the most challenging case with $m=15$ by $29.58$ points and $29.76$ points in $F1$-score on Wiki-ZSL and FewRel, respectively. 
It demonstrates the robustness of our model in solving the ZSRE effectively.
Besides, as \citeA{gong2021prompt} only reported their model performance on $m=15$, we also compare our model with ZS-SKA in this case. 
Specifically, our model improves the performance by $27.17$ points and $20.8$ points in $F1$-score on Wiki-ZSL and FewRel, respectively, compared to the state-of-the-art model ZS-SKA.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 2
\begin{table}[t]
\input{02table02.tex}
\hangcaption{Results on Wiki-ZSL and FewRel on different $m$ values. \textsuperscript{$\star$} indicates the results reported by \protect\citeA{chenli2021zs}; \textsuperscript{$\dagger$} marks the results we reproduced using the official source code of \protect\citeA{chenli2021zs}; --- indicates the results were not reported by \protect\citeA{gong2021prompt} in the cases of $m=5$ and $m=10$.}
\label{main_result}
\vspace{-0.9\Cvs}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The performance gain comes from the ability of our model to grasp the semantic correlation between instances and relations.
Indeed, our model was entirely designed for this goal in three ways.
($1$) Our model obtains high-quality relation representations.
($2$) The strategic design of mini-batches is aimed at semantic correlation learning.
($3$) Our approach constrains the consistency of the two-way interaction between instances and relations.
These aspects are discussed in detail in the following subsections.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Impact of Relation Representation}

We investigate the role of the relation representation quality in affecting the ZSRE task's performance.
Recall that, for Wiki-ZSL and FewRel datasets, both relation labels and descriptions are provided.


\citeA{chenli2021zs} exploited only relation descriptions and inputted them into the fixed pre-trained Sentence-BERT \cite{gurevych2019sentence} to obtain relation representations in their model ZS-BERT.
First, we attempt to follow this method to acquire such relation representations and use them in our model.
Table~\ref{tab_rel_rep} reports the experimental results. 
In all three different values of $m$, using the same fixed Sent-BERT encoder and only relation descriptions, our model still achieves better performance in $F1$ score by a large margin on both datasets. 
\pagebreak
For example, in the case of $m=15$, our model achieves $F1$ score of $40.31$ points and $42.52$ points on Wiki-ZSL and FewRel, respectively, thereby improving in $F1$ score by $4.59$ points and $4.46$ points on Wiki-ZSL and FewRel, compared to ZS-BERT by \citeA{chenli2021zs}.
However, using such fixed relation representations causes overfitting during training.
More severely, this hinders our model from grasping the semantic correlation between instances and relations effectively.

We, therefore, obtain relation representations via a learnable BERT-based encoder (Section~\ref{sec_rel_rep}) in our model.
Although relation labels and descriptions are available, \citeA{chenli2021zs} only used relation descriptions, while \citeA{gong2021prompt} only exploited relation labels to create relation representations.
Whereas relation labels provide concise and summary relation information, relation descriptions provide more detailed relation information.
Intuitively, they complement each other to yield the best relation representations.
We examine this intuition by feeding different inputs into the BERT-based encoder of our model to generate relation representations in all three different cases of $m$. 

In Table~\ref{tab_rel_rep}, using only the relation label with the learnable BERT-based encoder, our model also achieves an impressive performance in $F1$ score on both datasets in all three cases of $m$.
For instance, in the case of $m=15$, our model attains $53.47$ points and $57.34$ points in $F1$ score on Wiki-ZSL and FewRel, respectively.
It significantly enhances $F1$ score by $14.06$ points and $14.82$ points on Wiki-ZSL and FewRel, compared to using the fixed relation representations in our model.
This result proves the vital role of learning high-quality relation representations in solving the ZSRE.
Furthermore, we consider using only relation descriptions via the BERT-based encoder in our model in all three cases of $m$.
Compared with only relation labels, using only relation descriptions achieves better performance and improves the $F1$ score by a large margin on both datasets.
For example, when $m=15$, our model gains $F1$ score by $7.74$ and $5.35$ points on Wiki-ZSL and FewRel, respectively.
This may be reasonable because relation descriptions provide better relation representations with more detailed information.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[t]
\input{02table03.tex}
\hangcaption{Impact of the different relation representations in our model. \textsuperscript{$\dagger$} marks the results we reproduced using the official source code of \protect\citeA{chenli2021zs}. \textit{P}, \textit{R}, \textit{F1} stand for \textit{Precision}, \textit{Recall}, and \textit{F1-score} measures, respectively.}
\label{tab_rel_rep}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



Finally, using relation labels and descriptions to generate relation representations, our model achieves the best performance for Wiki-ZSL and FewRel in all different values of $m$.
For example, in the case of $m=15$, this combination improves the $F1$ score by $3.19$ points and $5.13$ points on Wiki-ZSL and FewRel, respectively, than using only relation descriptions.
We also observe significant improvements in $F1$ score by combining relation labels and descriptions on both datasets in the two remaining cases of $m=5$ and $m=10$.
This indicates that relation labels and relation descriptions complement each other to provide relation information more thoroughly, thereby acquiring the best relation representations leading to the best performance.      


In summary, we point out two important conclusions based on the extensive experiment results on all three different values of $m$ in Table~\ref{tab_rel_rep}. First, the quality of relation representations plays a vital role in improving the task performance of ZSRE. Second, to obtain high-quality relation representations, it is encouraged to use fine-tuned pre-trained language models instead of fixed ones in the training phase and combine both relation labels and descriptions to achieve the best representation. We expect the future works will consider these aspects in constructing relation representations for the ZSRE. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Impact of the Hyperparameter $K$}

As illustrated at the left part of Figure~\ref{our_model}, each prepared mini-batch for training our model has $K$ different relations and $K$ corresponding instances.
Such designed mini-batches facilitate the model in grasping the semantic correlation between instances and relations.
Thus, we inspect how the hyperparameter $K$ affects the system performance.


The relation numbers on the entire datasets Wiki-ZSL and FewRel are $113$ and $80$, respectively.
To enable the ZSRE scenario, we randomly select $m=15$ relations as \textit{unseen} relations, thereby splitting each dataset into the training and test sets.
Accordingly, the numbers of \textit{seen} relations in training on Wiki-ZSL and FewRel are $98$ and $65$.
Therefore, we try $K$ with several values in [$2$, $98$] on Wiki-ZSL and [$2$, $65$] on FewRel in the training stage.
At each value $K$, the reported $F1$ score is the average result obtained by repeating the experiment $5$ times for $5$ different random selections of $m$ ($m=15$) testing \textit{unseen} relations.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 2
\begin{figure}[b]
\begin{center}
\includegraphics{30-2ia2f2.pdf}
\end{center}
\caption{Impact of the hyperparameter $K$ in the case of $m=15$.}
\label{analysis_K}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Figure~\ref{analysis_K} shows the experimental results for the two test sets. 
Our model achieves the best performance with $K=5$ on Wiki-ZSL and FewRel, whereas it obtains the worst performance with $K=2$.
Interestingly, using the largest value $K$ (i.e., $K=98$ on Wiki-ZSL and $K=65$ on FewRel) does not give the best performance.
Conversely, compared to the best performance with $K=5$, using the largest value $K$ significantly decreases the performance in $F1$ score on both the datasets by $13.89$ points on Wiki-ZSL and $9.87$ points on FewRel. 
Obviously, when using a substantial value $K$ (e.g., $K=98$ on Wiki-ZSL), our model can easily be distracted from fully grasping the interaction between many relations and instances.
This proves that selecting a relevant value $K$ is essential to aid the model in effectively grasping the semantic correlation between instances and relations.
Meanwhile, previous studies \cite{chenli2021zs,gong2021prompt} only simply compared each instance to the total training \textit{seen} relations in learning the semantic relationship.
By contrast, our method of controlling the $K$ helps the model focus on thoroughly gripping the semantic correlation between instances and relations.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Impact of the Loss $\mathcal{L}_{KL}$}

We use the final objective function (Equation~\ref{eq_final_loss}) in the training stage to encourage the model to grasp the semantic interaction between instances and relations.
As defined in Equation~\ref{eq_final_loss}, the objective function comprises $\mathcal{L}_{CE}$ and $\mathcal{L}_{KL}$. 
While $\mathcal{L}_{CE}$ plays a significant role in learning the semantic correlation, $\mathcal{L}_{KL}$ also helps strengthen this goal by constraining two-way semantic distribution consistency.
Examining the necessity of using $\mathcal{L}_{KL}$ in our model, we attempt to remove $\mathcal{L}_{KL}$ from the final objective function.
The results are presented in Table~\ref{tab_kl_loss}.
Without using $\mathcal{L}_{KL}$, the system performance significantly decreases by $3.24$ and $3.21$ points in $F1$ score on Wiki-ZSL and FewRel, respectively.
These results reaffirm the vital role of the loss $\mathcal{L}_{KL}$ in supervising our model to grasp the semantic correlation between instances and relations, thereby solving the ZSRE effectively.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 4
\begin{table}[b]
\input{02table04.tex}
\caption{Impact of using the loss $\mathcal{L}_{KL}$ in our model in the case of $m=15$.}
\label{tab_kl_loss}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 3
\begin{figure}[b]
\begin{center}
\includegraphics{30-2ia2f3.pdf}
\end{center}
\caption{Impact of the hyperparameter $\alpha$ in the case of $m=15$.}
\label{analysis_alpha}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We further investigate the sensitivity of $\alpha$ to the task performance by changing different values of $\alpha$. As shown in Figure~\ref{analysis_alpha}, our model achieves the best performance with $\alpha=1$ on both Wiki-ZSL and FewRel datasets. Besides, when the value $\alpha$ is small (e.g., $\alpha=0.01$) or quite large (e.g., $\alpha=10$), it reduces the beneficial effect of using $\mathcal{L}_{KL}$ in improving the system performance.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Performance on Limited Labeled Data}

We further investigate the robustness of our model in solving the ZSRE under a limited labeled data scenario.
As described in Table~\ref{tab_statistics}, FewRel is a human-annotated balanced dataset consisting of $80$ relation types, each of which has $700$ instances.
First, we randomly split FewRel into training and test sets, where the training set includes $65$ seen relations and the test set consists of $15$ unseen relations.
We then fix the test set and change the rate of the labeled data to train the models.
Subsequently, the number of the seen relations of the training set is fine-tuned in [$10$, $65$].
Note that the experiment is repeated $5$ times for $5$ different random data divisions, and we report the average results.
We also run such experiments on ZS-BERT and compare it with our model under similar limited training data conditions because only the official source code\footnote{\url{https://github.com/dinobby/ZS-BERT}} of ZS-BERT is available.
Figure~\ref{analysis_lowsource} shows the experimental results.
Our model gains $2.0$ times higher accuracy than ZS-BERT in $F1$ score in all limited data cases.
This proves the robustness of our model in dealing with ZSRE under severely limited data conditions, which are popular in real-world scenarios.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 4
\begin{figure}[b]
\begin{center}
\includegraphics{30-2ia2f4.pdf}
\end{center}
\caption{Performance on the limited labeled data in the case of $m=15$.}
\label{analysis_lowsource}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Impact of Random Seed Sensitivity}

All previously reported experimental results are the average results obtained by running the experiment $5$ times with $5$ random $m$ selections ($m=15$).
We used the same fixed random seed in all of these experiments.
Thus, we further check the sensitivity of our model to different random seeds for system performance.


We first split the entire Wiki-ZSL dataset into a training set and a test set, where the training set includes $98$ seen relations and the test set consists of $15$ unseen relations.
Then, we try $5$ different random seeds to train our model and report the average testing results.
We repeat this process $3$ times and report all the results in Table~\ref{tab_random_seeds}.
Our model consistently outperforms ZS-BERT by a significant margin in the $F1$ score, all three times.
More importantly, based on the standard deviations of the $F1$ scores, our model is more stable than ZS-BERT when training with different random seeds.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 5
\begin{table}[t]
\input{02table05.tex}
\hangcaption{Impact of using different seeds to performance. The scores of ZS-BERT and our model are the average results of five runs with five different seeds. \textit{F1} score is in the format of \textit{mean} $\pm$ \textit{standard deviation}.}
\label{tab_random_seeds}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion and Future Work}

This paper presents a novel approach focusing entirely on enhancing the semantic correlation between instances and relations, which is key to solving ZSRE.
Our approach concentrates on three major aspects to achieve this goal: learning high-quality relation representations, designing purposeful mini-batches, and binding two-way semantic distribution consistency.
Extensive experiments on two benchmark datasets have demonstrated the effectiveness and robustness of our proposed model, particularly in limited training data scenarios.


However, our approach is tested on the two benchmark datasets in the general domain and might not work well in specialized domains like the biomedical field.
Therefore, we plan to evaluate our method in such domains in future work.
Besides, our proposed framework is related to dense retrieval models. Thus, we also plan to try other tricks in optimizing dense retrieval models, such as hard negatives and contrastive pre-training, for solving the zero-shot relation extraction task more effectively.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.7}
%%%%\bibliography{02refs}
\input{IA0300_02tran_bbl.tex}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Van-Hien Tran}{%
Van-Hien Tran received the B.E. degree in information technology from the University of Engineering and Technology, Vietnam National University (VNU), Vietnam, in 2014, the M.E. and Ph.D. degrees from the Nara Institute of Science and Technology (NAIST), Japan, in 2019 and 2022, respectively. Currently, he is a researcher at the National Institute of Information and Communications Technology (NICT). His research interests include machine learning and natural language processing.}


\bioauthor[:]{Hiroki Ouchi}{%
Hiroki Ouchi is an assistant professor at Nara Institute of Science and Technology (NAIST).
He received his M.S. and Ph.D. degrees from NAIST in 2015 and in 2018, respectively.
He was a postdoctoral researcher at RIKEN Center for Advanced Intelligence Project
(AIP) from 2018 to 2021.
His research interests cover natural language processing, machine
learning and geospatial information science.}

\bioauthor[:]{Hiroyuki Shindo}{%
Hiroyuki Shindo received the B.E. and
M.E. degrees from Waseda University, Japan,
in 2007 and 2009, respectively, and the Ph.D.
degree in engineering from the Nara Institute of Science and Technology (NAIST), Ikoma, Japan, in 2013. From 2009 to 2014, he was a Researcher with NTT Communication Science Laboratories. He is currently an Associate Professor with the Graduate School of Information Science, NAIST. His research interests include machine learning and computational linguistics.}


\bioauthor[:]{Yuji Matsumoto}{%
Yuji Matsumoto is currently a team leader at RIKEN Center for Advanced
Intelligence Project (AIP).  He received his M.S. and Ph.D. degrees in
information science from Kyoto University in 1979 and in 1990, respectively.
He joined Machine Inference Section of Electrotechnical Laboratory in
1979.  He has been an academic visitor at Imperial College
of Science and Technology, the deputy chief of First Laboratory at
ICOT, an associate professor at Kyoto University, and a professor at
Nara Institute of Science and Technology.  His main research interests
are natural language understanding and machine learning.  He is an ACL
fellow and a fellow of Information Processing Society of Japan.}


\bioauthor[:]{Taro Watanabe}{%
Taro Watanabe received the B.E. and M.E. degrees in information science from Kyoto Univ., Kyoto, Japan in 1994 and 1997, respectively, and obtained the Master of Science degree in language and information technologies from the School of Computer Science, Carnegie Mellon University in 2000.
In 2004, he received the Ph.D. in informatics from Kyoto Univ., \linebreak
Kyoto, Japan.
After working as a researcher at ATR, NTT and NICT, and as a software engineer at Google, he is a professor at Nara Institute of Science and Technology starting from 2020.
His research interests include natural language processing, machine learning and machine translation.}


\end{biography}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\biodate



\end{document}

