\documentclass[english]{jnlp_3.5}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{hangcaption_jnlp}
%%%%\usepackage{udline}
%%%%\setulminsep{1.2ex}{0.2ex}
%%%%\let\underline=\ul
\allowdisplaybreaks

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
%%%%% \usepackage{url}
%%%%% \usepackage{amsmath,amssymb}
%%%%\usepackage{amssymb}
\usepackage{bm}
%%%%% \usepackage{graphicx}
%%%%\usepackage{subfigure}
\usepackage{multirow}
\usepackage{makecell}
\usepackage{pifont}
%%%%% \usepackage{color,soul}
\usepackage[svgnames]{xcolor}
\usepackage{arydshln}
\usepackage{booktabs}
%%%%% \renewcommand{\UrlFont}{\ttfamily\small}
\newcommand{\argmax}{\mathop{\rm arg~max}\limits}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{29}
\Number{1}
\Month{March}
\Year{2022}
%%%%\Issuetitle{Special Issue: XXXXXXXX}

\received{2021}{7}{7}
\revised{2021}{10}{26}
%%%%\rerevised{2006}{5}{6}
\accepted{2021}{12}{6}

\setcounter{page}{187}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Named Entity Recognition and Relation Extraction Using Enhanced Table Filling by Contextualized Representations}
\eauthor{Youmi Ma\affiref{titech} \and Tatsuya Hiraoka\affiref{titech} \and Naoaki Okazaki\affiref{titech}} 
\eabstract{%
In this study, we propose a method designed  to extract named entities and relations from unstructured text based on table representations. To extract named entities, the proposed method computes representations for entity mentions and long-range dependencies using contextualized representations without hand-crafted features or complex neural network architectures. To extract relations, it applies a tensor dot product to predict all relation labels simultaneously without considering dependencies among relation labels. These advancements significantly simplify the proposed model and the associated algorithm for the extraction of named entities and relations. Despite its simplicity, the experimental results demonstrate that the proposed approach outperformed the state of the-art methods on multiple datasets. Compared with existing table-filling approaches, the proposed method achieved high performance solely by independently predicting the relation labels. In addition, we found that incorporating dependencies of relation labels into the system obtained little performance gain, indicating the effectiveness and sufficiency of the tensor dot-product mechanism for relation extraction in the proposed architecture. Experimental analyses were also performed to explore the benefits of joint training with named entity recognition in relation extraction in our design. We concluded that joint training with named entity recognition assists relation extraction to improve the span-level representation of entities. 
}
\ekeywords{Information Extraction, Named Entity Recognition, Relation Extraction, Contextualized Word Embedding}

\headauthor{Ma et al.}
\headtitle{NER and RE Using Enhanced Table Filling by Contextualized Representations}

\affilabel{titech}{}{Tokyo Institute of Technology}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}
\label{sec:intro}

Named entity recognition (NER) \cite{Nadeau:07,ratinov-roth-2009-design} and relation extraction (RE) \cite{zelenko-etal-2002-kernel,zhou-etal-2005-exploring} are two major sub-tasks of information extraction (IE).
Recent studies have reported on the advantages of performing these two tasks jointly in terms of both efficiency and accuracy \cite{miwa-sasaki-2014-modeling,li-ji-2014-incremental,gupta-etal-2016-table,miwa-bansal-2016-end,zhang-etal-2017-end}. Compared with pipelined approaches \cite{chan-roth-2011-exploiting}, the joint extraction of named entities (NE) and relations enables models to capture dependencies between entities and relations \cite{li-ji-2014-incremental,wang-lu-2020-two}.

Several existing methods have been developed to perform the joint extraction of named entities (NEs) and relations as a table-filling problem, wherein entity and relation labels are represented as cells in a single table \cite{miwa-sasaki-2014-modeling,gupta-etal-2016-table,zhang-etal-2017-end,wang-lu-2020-two}. As shown in Figure~\ref{fig:example_intro}, for a given natural language sentence, the label space of NER and RE is represented by assigning NE labels along with the diagonal  entries of the table, and relation labels along with the off-diagonal entries of the table. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1
\begin{figure}[b]
\begin{center}
\includegraphics{29-1ia8f1.pdf}
\end{center}
\hangcaption{Example of the table-filling strategy for the sentence ``Johanson Smith lives in London''. $\perp$ indicates a negative relation label (that is, no relation exists among the corresponding words). }
\label{fig:example_intro}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Although table-filling representations have been reported to be effective, approaches of this type require careful design of features and search strategies to ensure high performance. Previous studies have designed complicated features to encode contexts and long-range dependencies between NE and relations. For example, to properly determine features, \citeA{miwa-sasaki-2014-modeling} used handcrafted syntactic features (such as the shortest path between two words in a syntactic tree), and \citeA{zhang-etal-2017-end} extracted syntactic information using the encoder of a pretrained syntactic parser. To properly determine the search order, \citeA{miwa-sasaki-2014-modeling} explored six different decoding (search) strategies to fill a table based on historical predictions. \citeA{wang-lu-2020-two}, one of the current state-of-the-art table-filling systems, also applied separate groups of encoders to represent NEs and relations and explored several patterns for processing table information.  To summarize, there are two limitations of the aforementioned existing designs: the performance of such models depends heavily on the expressiveness of pre-defined features, and the exploration of decoding strategies is time-consuming and computationally intensive. 

To address these issues, we present a method designed to jointly perform NER and RE by enhancing the table-filling approach. Motivated by the expressiveness of deep contextualized word representations pre-trained on large-scale text corpora \cite{peters-etal-2018-deep,devlin-etal-2019-bert}, we utilize Bidirectional Encoder Representations from Transformers (BERT, \citeA{devlin-etal-2019-bert}) contextualized representations initialized with pre-trained weights to represent entity mentions and encode long-range dependencies among entities to simplify feature engineering. Furthermore, the proposed model enhances entity representations with span-based features. We utilize a tensor dot product to fill cells of relation labels in the table all-at-once (instead of cell-by-cell with beam search) to reduce the burden of exploring search strategies. This modification also simplifies the decoding process and improves decoding parallelism by performing RE using matrix and tensor operations.

For evaluation, we tested the proposed models on three benchmark datasets, including 
\linebreak
CoNLL04 \cite{roth-yih-2004-linear}, ACE05\footnote{\url{https://catalog.ldc.upenn.edu/LDC2006T06}} (English), and Wet Lab Protocol Corpus (WLPC) \cite{kulkarni2018wetlab} for both NER and RE. Based on the experimental results, the performance of the proposed method was found to be comparable to that of previous state-of-the-art methods, including \citeA{wang-lu-2020-two} and \citeA{lin-etal-2020-joint}, despite its simplicity. Note that although we trained and evaluated our models using English-language datasets, the proposed method is language agnostic and can be applied to NER and RE tasks in other languages. 

In addition, experimental analyses were performed to better understand the behavior of the proposed system. After exploring several predefined search orders, we confirmed that the decoding strategies had little effect on the performance of the model. This finding indicates that given contextualized representations encoded with long-range dependencies, the tensor dot-product mechanism enabled to predict relation labels directly, without incorporating any dependencies between relation labels. Moreover, we experimentally measured the benefits of joint training and prediction between NER and RE and concluded that joint training and prediction can be mutually beneficial for these two tasks under the proposed architecture. 

The contributions of this study are summarized as follows. 
\begin{itemize}
  \item We propose a simple but effective table-filling model designed to jointly extract entities and relations using BERT-based contextualized representations. In contrast to existing table-filling approaches \cite{miwa-sasaki-2014-modeling,zhang-etal-2017-end,gupta-etal-2016-table,wang-lu-2020-two}, the model requires less effort for designing features and search strategies. Experimental analyses were conducted to examine the behavior of the proposed model.\footnote{The source code  is publicly available at \url{https://github.com/YoumiMa/TablERT}.}
    \item We verify that predicting relation labels all-at-once via the dot-product mechanism yielded reasonable performance on RE under the proposed architecture. That is, the model extracts relations efficiently, even when discarding dependencies between relation labels. The proposed model can thus serve as a strong baseline to model pairwise relationships on top of pre-trained transformers.
\end{itemize}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Work}
\label{related work}

\vspace{-0.5\Cvs}
Previous studies have performed joint extraction of entities and relations as a structured prediction problem using global features and predefined search algorithms.
\citeA{li-ji-2014-incremental} presented an incremental algorithm for joint NER and RE using global features and inexact (beam-search) decoding.
\citeA{miwa-sasaki-2014-modeling} proposed a table representation for entities and relations. Furthermore, they investigated hand-crafted features and complex search heuristics on the table.
\citeA{gupta-etal-2016-table} enhanced the table-filling approach by adapting recurrent neural networks (RNNs) to fill table cells in a pre-defined sequential order.
\citeA{miwa-bansal-2016-end} explored a shared representation for entities and relations by stacking bidirectional tree-structured and sequential Long Short Term Memory based Recurrent Neural Networks (LSTM-RNNs).
\citeA{zhang-etal-2017-end} integrated a global optimization technique and syntax-aware word representations.
The abovementioned studies relied heavily on feature engineering (such as hand-crafted features or specialized deep neural network models) and search/optimization strategies. In comparison, our method avoids these engineering efforts while obtaining high performance by utilizing deep contextualized word representations.

In recent years, deep contextualized word representations have been widely used to address the sequential labeling problem.
\citeA{liu-etal-2019-gcdt} proposed a deep transition architecture enhanced by the global context and reported improvements on entity extraction and chunking tasks using contextualized word embeddings.
\citeA{strakova-etal-2019-neural} demonstrated the effectiveness of contextualized representations on architectures for nested named entity recognition, where the NE may overlap with the multiple labels assigned. Their study, from which we gained insights useful to the design of the proposed system, proved that proper utilization of contextualized representations improves the accuracy of information extraction.


More recently, span enumeration methods have gained popularity for the joint extraction of entities and relations \cite{luan-etal-2019-general,Wadden2019EntityRA,spert,lin-etal-2020-joint}.
In general, span enumeration methods inspect possible entity spans for an input sentence based on certain criteria such as the maximum number of words and choose likely spans using features extracted from the span candidates.
\citeA{luan-etal-2019-general} proposed a general framework of information extraction called DyGIE, which can incorporate global information on a dynamic span graph. 
\citeA{Wadden2019EntityRA} further expanded the model to a multi-task framework for extracting entities, relations, and events. The method received multiple sentences from the same document as an input and enumerated candidate spans for relation, co-reference, and event identification. To update the span representations of entities, they carefully designed strategies for dynamic graph construction and span refinement. 
Following their study, \citeA{lin-etal-2020-joint} explicitly modeled cross-task and cross-instance dependencies by introducing global features following a set of global feature templates.


In addition to these graph-based approaches, \citeA{spert} also proposed an end-to-end RE model designed to extract both entities and relations. The method utilized pre-trained BERT models and enumerated candidates for entity spans. Using a negative sampling strategy for both NER and RE, the method classified entity and relation candidates as positive or negative. 
Based on this study, \citeA{ji-etal-2020-span} incorporated span-specific and contextualized attention for calculating semantic representations. Their study further emphasized the importance of reducing the rate and severity of span (entity) classification errors .


Although span enumeration is popularly used in joint NER and RE, non-span enumeration methods also have the potential to exhibit good performance in this regard. \citeA{wang-lu-2020-two} introduced a system showing state-of-the-art performance on multiple NER and RE benchmarks using a non-span enumeration approach. In their design, entities and relations were encoded by table-sequence encoders, with entities encoded by sequence representations and relations by table representations. They also leveraged attention weights from pre-trained language models such as BERT \cite{devlin-etal-2019-bert} and ALBERT \cite{Lan2020ALBERT} to convey word--word interaction information. The proposed system, however, is simpler and more lightweight than \citeA{wang-lu-2020-two} while performing comparably.


Meanwhile, pipelined approaches have proven promising. Specifically, \citeA{zhong-chen-2021-frustratingly} recently introduced a pipelined approach for extracting first entities and then relations. Their RE model fully utilized entity label information by re-encoding entity labels into contextualized representations. While \citeA{zhong-chen-2021-frustratingly} learned distinct representations for entities and relations, other studies have shown that joint training of these tasks can be mutually beneficial \cite{miwa-bansal-2016-end,jiang-etal-2020-generalizing}. We follow the latter line of study to investigate the benefits of joint training.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Proposed Method}
\label{method}

The proposed approach is designed to extract NEs and relation instances. Given a sequence of words $w_1 w_2 \cdots w_n$, where $n$ is the number of words in the input, our goal is to extract relation triples in the form of $\mathrm{(arg0_{type0}, relation, arg1_{type1})}$. Here, $\mathrm{type0}$ represents the NE type of the mention $\mathrm{arg0}$; $\mathrm{arg1}$ and $\mathrm{type1}$ are defined analogously.
We define $\mathcal{E}$ and $\mathcal{R}$ as the label sets of the NEs and relations, respectively.

The table representation \cite{miwa-sasaki-2014-modeling} is employed to jointly recognize NEs and relation instances. Formally, we define an $n \times n$ upper triangular matrix $\bm{Y}$, where the diagonal element $\bm{Y}_{i,i} \in \mathcal{E}$ ($1 \leq i \leq n$) represents an NE label for the word $w_i$, and an off-diagonal element $\bm{Y}_{i,j} \in \mathcal{R}$ ($1 \leq i < j \leq n$) represents a directed relation label between the words $w_i$ and $w_j$. Following \citeA{zhang-etal-2017-end}, we hard-code directions into relation labels $\mathcal{R}$ to avoid considering the lower triangular part of the table for RE. Thus, our model can be viewed as a mapping from a sequence of words $w_1 w_2 \cdots w_n$ to an upper triangular matrix $\bm{Y}$. For simplicity, we denote an NE label as $y_i = \bm{Y}_{i,i}$.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 2
\begin{figure}[b]
\begin{center}
\includegraphics{29-1ia8f2.pdf}
\end{center}
\hangcaption{Example of the upper triangular matrix $\bm{Y}$ and table-filling
strategy. Numbers in cells indicate the order of cell filling (decoding). $\perp$ indicates a negative relation label (that is, no relation exists among the corresponding words). }
\label{fig:example}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Figure~\ref{fig:example} illustrates an example of a matrix $\bm{Y}$ for the input sentence, ``Johanson Smith lives in London''. Notably, the relations are mapped from 1-dimensional word sequences to a 2-dimensional matrix $\bm{Y}$ on the entity-level. Further, each word inside an entity span is annotated with the corresponding relation label. For example, consider the sentence shown in Figure~\ref{fig:example}. For the NE ``Johanson Smith'' labeled as \textsc{Person}, relation $\overrightarrow{\textsc{LiveIn}}$ is labeled on both $\bm Y_{1,5}$ and $\bm Y_{2,5}$ corresponding to ``Johanson'' and ``Smith'', respectively. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 3
\begin{figure}[t]
\begin{center}
\includegraphics{29-1ia8f3.pdf}
\end{center}
\hangcaption{Named Entity Recognition model. For clarity, we only show the calculation of the entity hidden vectors $\bm{h}_1^{\rm (ent)}$ and $\bm{h}_3^{\rm (ent)}$ for the words ``Johanson and ``lives'' in detail.}
\label{fig:ner}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

This study is based on the application of pre-trained language models to leverage contextual information to solve NER and RE. The proposed method stacks layers for NER and RE on top of a BERT encoder by default.
As illustrated in Figures~\ref{fig:ner} and~\ref{fig:re}, the proposed method computes word representations from contextualized embeddings of subword tokens obtained by byte pair encoding (explained in \S~\ref{sec:alignment}) and performs NER (\S~\ref{sec:ner}) and RE (\S~\ref{sec:re}).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 4
\begin{figure}[t]
\begin{center}
\includegraphics{29-1ia8f4.pdf}
\end{center}
\hangcaption{Relation Extraction model. The relation word representation $\bm{h}_i^{\rm (rel)}$ is computed from the NE module. In the right-most table, $-$ indicates a masked-out cell during both training and prediction.}
\label{fig:re}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Word Representations}
\label{sec:alignment}

The BERT tokenizer uses WordPiece to split words (such as ``Johanson'') into subword tokens (such as, ``Johan'' and ``\#\#son'') with the aid of byte pair encoding. This technique has been proven to be effective in reducing the size of vocabulary and unknown words \cite{devlin-etal-2019-bert}. Because NEs are annotated at the word level, we must generate word-level representations from subword-level representations during both training and prediction.

In this study, we compute the max-pooling of BERT embeddings of subword tokens composing the word as its representation\footnote{We examine the performance of the CoNLL04 development set using (1) embedding of first subword tokens  \cite{devlin-etal-2019-bert}, (2) mean pooling of constituent subword tokens, (3) mean pooling of constituent subword tokens and \texttt{[CLS]}, and (4) max pooling of constituent subword tokens. Among these, max pooling exhibits the best performance.} \cite{liu-etal-2019-gcdt}.
\begin{equation}
    \bm{e}_{w_i} = f(\bm{e}_{t_{i,1}}, \bm{e}_{t_{i,2}}, \cdots, \bm{e}_{t_{i,s}}) .
\end{equation}
Here, we assume that the word $w_i$ comprises $s$ subword tokens $t_{i,1}, t_{i,2},$ $\cdots, t_{i,s}$; $\bm{e}_{w_i}$ and $\bm{e}_{t_{i,k}}$ are embeddings for the word $w_i$ and subword token $t_{i,k}$, respectively. $f$ is used to denote a max pooling operation.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Named Entity Recognition}
\label{sec:ner}

We use the BILOU (beginning, inside, last, outside, unit-length) notation to represent the spans of NEs \cite{ratinov-roth-2009-design}.
We consider NER as a sequential labeling task, where each word $w_i$ in the input is labeled as $y_i$ (a diagonal element in $\bm{Y}$) in the BILOU notation.
Inspired by \citeA{zhang-etal-2017-end}, wherein span representations are extracted from bidirectional LSTM cells as features, we enhance the existing architecture using span features from previous time steps.

Specifically, the model predicts an NE label for the word $w_i$ based on three features: (1)  representation $\bm{e}_{w_i}$ of the word $w_i$, (2) embedding $\bm{l}_{y_{i-1}}$ of the label $y_{i-1}$ at the previous time step $i-1$, and (3) max pooling of BERT embeddings of the previous NE span appearing at time steps $({\rm first}(i-1), \cdots, i-1)$. Here, ${\rm first}(i)$ denotes the timestep at which the NE span including word $i$ starts. For example, when processing the sentence shown in Figure \ref{fig:ner}, because  the phrase ``Johanson Smith'' is labeled as an NE, we have ${\rm first}(1)={\rm first}(2)=1$. Similarly, because ``lives'' is a single non-entity word, we have ${\rm first}(3)=3$. Following \citeA{zhang-etal-2017-end}, when the previous word is labeled as \texttt{O}, we assume that the previous span is of a unitlength, that is, ${\rm first}(i-1) = i-1$. In addition, when the timestep is one ($i=1$), we set the previous word representation as a zero vector and the previous label as a unit length outside (\texttt{O}), that is, 
$y_0 = \text{\tt O}$ and $\bm{e}_{w_0} = \bf{0}$. 

We define the entity representation for predicting the label of the current timestep $i$ as the concatenation of the three features described above.
\begin{equation}
    \bm{h}_i^{\rm (ent)} = \bm{e}_{w_i} \oplus \bm{l}_{y_{i-1}} \oplus f(\bm{e}_{w_{{\rm first}(i-1)}}, \cdots, \bm{e}_{w_{i-1}}) ,
\label{eq:entity_repr}
\end{equation}
where $\oplus$ denotes a vector concatenation. $\bm{l}_{y_{i-1}}$ is retrieved from the $y_{i-1}$-th row of a randomly initialized trainable lookup table $\bm{L}\in\mathbb{R}^{|\mathcal{E}| \times d_{\rm ent}}$ storing embeddings of all NE labels, where $d_{\rm ent}$ represents the number of dimensions of the NE label embeddings.

We apply a fully connected layer followed by a softmax function to obtain the probability distribution across all possible NE labels at time step $i$, as given below.
\begin{align}
 & \bm{\Phi}_i^{\rm (ent)} = \bm{W}^{\rm (ent)} \bm{h}_i^{\rm (ent)} + \bm{b}^{\rm (ent)}, \label{eq:ne_score}\\
 & \hat{\bm{y}}_{i} = \sigma(\bm{\Phi}_i^{\rm (ent)}).\label{eq:ne_prediction}
\end{align}
Here, $\bm{W}^{\rm (ent)}$ and $\bm{b}^{\rm (ent)}$ represent the matrix and the bias vector for a linear transformation, and $\bm{\Phi}^{\rm (ent)} \in \mathbb{R}^{n \times |\mathcal E|}$ represents the score matrix. By applying the softmax function $\sigma$ on the i-th row of the matrix $\bm{\Phi}^{\rm (ent)}$, we obtain the vector $\hat{\bm{y}}_{i}$ representing the probability distribution over NE labels, and then fill the element $\bm{Y}_{i,i}$ ($=y_i$) with the NE label yielding the highest probability for $\hat{\bm{y}}_{i}$. 
Thus, we perform NER by filling up the diagonal elements $\bm{Y}_{i,i}$ from $i=1$ to $n$.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Relation Extraction}
\label{sec:re}

We perform RE on top of the BERT encoder and entity spans recognized in \S~\ref{sec:ner}.

After the NER model fills all the diagonal elements in $\bm{Y}$, the RE model predicts all off-diagonal elements in $\bm{Y}$.
We adapt a tensor dot-product to score each word pair along with the relation label distribution. The computation is similar to the multi-head self-attention mechanism \cite{vaswani2017attention}; however, our goal is not to compute attention weights over entity representations.\footnote{We also considered the deep bi-affine attention mechanism used in \citeA{dozat2017deep} and \citeA{NguyenV_ECIR2019}. Nevertheless, we found the use of the tensor dot product to be sufficient during the early stages of the experiments.}

Our model utilizes the features of both words and NE labels composing an entity to obtain relation representations to predict relation labels.
Let ${\rm last}(i)$ denote the time step wherein the entity span containing the time step $i$ ends, analogous to ${\rm first}(i)$ defined in \S~\ref{sec:ner}. As an example, for the sentence delineated in Figure~\ref{fig:ner}, we have ${\rm last}(1) = {\rm last}(2)=2$. The word-span feature $\bm{z}_i$ (at time step $i$) is computed using the representations of the constituent words in the entity span, as given below.
\begin{equation}
    \bm{z}_i = f(\bm{e}_{w_{{\rm first}(i)}}, \cdots, \bm{e}_{w_{i}}, \cdots, \bm{e}_{w_{{\rm last}(i)}}) .
    \label{eq:pred_z}
\end{equation}
Thus, $\bm{z}_i$ is computed from the max pooling operation performed across word representations of the entity span starting at ${\rm first}(i)$ and ending at ${\rm last}(i)$.
Max pooling across embeddings of the NE labels $\bm{l}_{y_{{\rm first}(i)}}, \cdots, \bm{l}_{y_i} , \cdots,$ $\bm{l}_{y_{{\rm last}(i)}}$ is then used as the label-span feature at time step $i$.
Mathematically, the word representation, as an input to the RE model, is a concatenation of the word- and label-span features, as expressed below. 
\begin{equation}
    \bm{h}_i^{\rm (rel)} = \bm{z}_i \oplus f(\bm{l}_{y_{{\rm first}(i)}}, \cdots, \bm{l}_{y_i} , \cdots, \bm{l}_{y_{{\rm last}(i)}}) 
    \label{eq:hrel}.
\end{equation}

For each possible relation $r \in \mathcal{R}$, we apply linear transformations parameterized by two matrices $\bm{W}^{(q)}_{r}, \bm{W}^{(k)}_{r} \in \mathbb{R}^{d_{\rm att} \times d_{\rm rel}}$ and two bias vectors $\bm{b}^{(q)}_r, \bm{b}^{(k)}_r \in \mathbb{R}^{d_{\rm att}}$.
\begin{align}
    & \bm{q}_{i,r} = \bm{W}^{(q)}_{r} \bm{h}_i^{\rm (rel)} + \bm{b}^{(q)}_{r} ,\label{eq:q} \\
    & \bm{k}_{i,r} = \bm{W}^{(k)}_{r} \bm{h}_i^{\rm (rel)} + \bm{b}^{(k)}_{r} ,\label{eq:k}. 
\end{align}
where $d_{\rm rel}$ denotes the number of dimensions of $\bm{h}_i^{\rm (rel)}$, and $d_{\rm att}$ represents the number of dimensions after the transformation. $\bm{q}_{i,r} \in \mathbb{R}^{d_{\rm att}}$ and $\bm{k}_{i,r} \in \mathbb{R}^{d_{\rm att}}$ are query and key vectors, respectively, for the relation $r$ at time step $i$.
As demonstrated, we map the word representation vector $\bm{h}_i^{\rm (rel)}$ into the query and key spaces associated with the relation $r$.

After collecting both query and key vectors for all the relations $r \in \mathcal{R}$ at all time steps $i \in \{1, \dots, n\}$, we obtain two tensors $\mathsf{Q} \in \mathbb{R}^{n \times |\mathcal{R}| \times d_{\rm att}}$ and $\mathsf{K} \in \mathbb{R}^{n \times |\mathcal{R}| \times d_{\rm att}}$.
Slices of the tensors are as follows.
\begin{align}
    & \mathsf{Q}_{i,r,:} = \bm{q}_{i,r} = \bm{W}^{(q)}_{r} \bm{h}_i^{\rm (rel)} + \bm{b}^{(q)}_{r} ,\\
    & \mathsf{K}_{i,r,:} = \bm{k}_{i,r} = \bm{W}^{(k)}_{r} \bm{h}_i^{\rm (rel)} + \bm{b}^{(k)}_{r} .
\end{align}

We compute a probability distribution across all possible relations for every combination of $i$ and $j$ ($1 \leq i < j \leq n)$, realized by the dot product of $\mathsf{Q}$ and $\mathsf{K}$, followed by a softmax function,
\begin{align}
 & \mathsf{\Phi}_{i,j,:}^{\rm (rel)} = (\mathsf{Q}\mathsf{K}^\top)_{i,j,:}, \label{eq:rel_dotp}\\
    & \hat{\bm{Y}}_{i,j} = \sigma(\mathsf{\Phi}_{i,j,:}^{\rm (rel)}) .
    \label{eq:rel_class}
\end{align}
Here, $(\cdot)_{i,j,:}$ denotes a slice (vector) of the tensor that extracts all elements in the last dimension, with the first dimension indexed $i$ and the second dimension indexed $j$.

$\mathsf{\Phi}^{\rm (rel)} \in \mathbb{R}^{n \times n \times |\mathcal R|}$ represents the score tensor, and the softmax function $\sigma$ computes the probability distribution $\hat{\bm{Y}}_{i,j}$ across all relation labels $\mathcal{R}$ for an arbitrary word pair $(i,j)$.
We fill in cell $\bm{Y}_{i,j}$ with the relation label yielding the highest probability for $\hat{\bm{Y}}_{i,j}$.
Thus, the RE model predicts the relation labels for all pairs of input words at once using Equation~\ref{eq:rel_class} on the top of NE labels and spans predicted by the NE module.

Meanwhile, we replace Equation~\ref{eq:hrel} with Equation~\ref{eq:hrel-train} during the training phase.
\begin{equation}
    \bm{h}_i^{\rm (rel)} = \bm{e}_{w_i} \oplus  \bm{l}_{y_i} .
\label{eq:hrel-train}
\end{equation}
This replacement is performed because Equation~\ref{eq:hrel} repeats the same pattern of parameter updates for multiple words within an entity span as a result of the max pooling operation.
In contrast, Equation~\ref{eq:hrel-train} promotes different patterns of parameter updates for different words in an entity span.\footnote{We explore several combinations (such as, using Equation~\ref{eq:hrel} for both training and prediction phases). Among all the combinations, that of Equation~\ref{eq:hrel} for prediction and Equation~\ref{eq:hrel-train} for training yielded the best performance.}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Training and Prediction}
\label{sec:training}

The objective of the training process is to minimize the sum of the cross-entropy losses of the NER ($\mathcal{L}^{\rm(ent)}$) and RE ($\mathcal{L}^{\rm(rel)}$),
\begin{equation}
    \mathcal{L} = \mathcal{L}^{\rm(ent)} + \mathcal{L}^{\rm(rel)} .
    \label{eq:loss}
\end{equation}


During training, the proposed NER model uses ground-truth NE labels and spans, similar to \citeA{zhang-etal-2017-end}. The proposed RE model uses NE labels and spans predicted from the NER model. Specifically, we perform a greedy search (from left to right) to obtain a sequence of NE labels and pass the sequence to the RE model. Our RE model then utilizes the NE labels and spans indicated by the label sequence to obtain the relation logits.

Notably, as shown in Figure~\ref{fig:example}, if a phrase is labeled as an NE, relations originating from or pointing to the phrase are mapped to the matrix $\bm Y$ on the span level. The mapping strategy enables the model to fully update parameters for different words inside an entity span during relation training. During prediction, we ignore the inconsistency of label predictions among componential words by max-pooling according to Equation~\ref{eq:hrel}, which results in the same word- and label-span features for all words inside the span. 

The second difference between the training and prediction processes is that in the latter, the greedy search is replaced by a beam search to enable interaction between NER and RE models. Our beam search considers both the NE predictions and relation predictions. To briefly summarize, when the beam size is $K$, we first retain the candidates with top-$K$ scores during NE decoding, then perform RE predictions on each candidate to obtain the relation score and select the candidate yielding the highest overall score as the final prediction. The following sections describe the search process in detail.


For a sentence of length $n$, the NE scores of the candidates are computed by $n$ incremental steps of the beam search. At the $t$-th step, for a candidate $\bm{y}_{1:t} = y_1, y_2, \cdots, y_t$, where $y_i$ denotes the predicted NE label of the $i$-th word, the NE score $S^{\rm (ent)}$ is computed as
\begin{equation}
S^{\rm (ent)}(\bm{y}_{1:t}) = \sum_{i=1}^t \bm{\Phi}^{\rm (ent)}_{i,y_i}.
 \label{eq:score_ne}
\end{equation}
Here, $\bm{\Phi}^{\rm (ent)}_{i, y_i}$ represents the score of the NE label $y_i$ predicted at the $i$-th step. We record $K$ candidates $\{\bm{y}_{1:t}^{(1)}, \bm{y}_{1:t}^{(2)}, \cdots, \bm{y}_{1:t}^{(K)}\}$ composing a candidate set $C_{1:t}$ to prune the search space.

At each incremental step, the candidate set $\mathcal{C}_{1:t+1}$ is computed from the existing candidate set $\mathcal{C}_{1:t}$. We collect the candidates with the top-$K$ scores at each step as follows.
\begin{equation}
\mathcal{C}_{1:t+1} = {\rm K }\argmax_{\substack{\bm{y}_{1:t} \in \mathcal{C}_{1:t}, \\ y_{t+1} \in \mathcal{E}}}S^{\rm (ent)}(\bm{y}_{1:t} \oplus y_{t+1}),
\end{equation}
where ${\rm K}\argmax$ denotes $K$-arguments of the maxima\footnote{In other words, ${\rm K}\argmax_{x\in S} f(x)$ defines $K$ arguments $\{x_1,x_2,\cdots,x_K\}$ for which $f(x_K)$ attains the $K$-th highest value  over domain $S$.} and $\oplus$ denotes concatenation. After $n$ steps, we obtain a set $\tilde{\mathcal{C}}$ containing the best $K$ candidates, each of which is an NE label sequence $\bm{\tilde{y}}$ of length $n$, as follows.
\begin{equation}
\tilde{\mathcal{C}} = \{\bm{\tilde{y}}^{(1)}, \bm{\tilde{y}}^{(2)}, \dots, \bm{\tilde{y}}^{(K)}\} = \mathcal{C}_{1:n} = \{\bm{y}_{1:n}^{(1)}, \bm{y}_{1:n}^{(2)}, \dots, \bm{y}_{1:n}^{(K)}\}.
\end{equation}



We then compute the relation score for each candidate. To do so, we first obtain the relation score tensor $\mathsf{\Phi}^{\rm (rel)}$ of each candidate. However, as shown in Equation~\ref{eq:rel_dotp}, the relation scores of word pairs $\mathsf{\Phi}^{\rm (rel)}$ are computed using the dot product of linearly transformed word representations. In contrast, the NE scores of words $\bm{\Phi}^{\rm (ent)}$ are directly computed by linear transformation, as shown in Equation~\ref{eq:ne_score}. As a result, a magnitude difference may be observed between the NE scores and relation scores. Specifically, each entry in the NE score matrix $\bm{\Phi}^{\rm (ent)}$ is the square root of that in the relation score tensor $\mathsf{\Phi}^{\rm (rel)}$, with regard to the magnitude. Therefore, to eliminate this difference in magnitude, we heuristically ``normalize'' each relation score by dividing it by the square root of its absolute value as follows.
\begin{equation}
 \overline{\mathsf \Phi}^{\rm (rel)}_{i,j,r} = \frac{{\mathsf \Phi}^{\rm (rel)}_{i,j,r}}{\sqrt{|{\mathsf \Phi}^{\rm (rel)}_{i,j,r}}|}.
 \label{eq:re_normalization}
\end{equation}

For each off-diagonal cell $(i,j)$, instead of summing the scores of all possible relation labels, we choose the relation label yielding the maximum score as its hypothesis and the corresponding score as its score. Taking the $K$-th candidate as an example, $\tilde{\bm{y}}^{(K)} = \bm{y}_{1:n}^{(K)}$ is expanded to $\tilde{\bm{Y}}^{(K)}$ with the relation labels at cells above the diagonal, that is,
\begin{equation}
\bm{\tilde{Y}}^{(K)}_{i,j} = \begin{cases}
\bm{\tilde{y}}_{i}^{(K)} & (i = j), \\
\argmax_{r' \in \mathcal{R}} \overline{\mathsf \Phi}^{\rm (rel)}_{i,j,r'} & (i < j).
\end{cases}
 \label{eq:re_r}
 \end{equation}

The relation score of each candidate is then defined as the summation of relation scores for all off-diagonal cells. Notably, for a sentence with $n$ words, the amounts of diagonal and off-diagonal cells are $\Theta(n)$ and $\Theta(n^2)$, respectively. Therefore, even after the ``normalization'' of scores on off-diagonal cells similar to Equation~\ref{eq:re_normalization}, the relation score, which is the sum of all off-diagonal cells, and the NE score, which is the sum of all diagonal cells, remains unbalanced because of this difference in the number of entries. Hence, we divide the relation scores by $n$ as follows for better balancing when computing the overall score.
\begin{equation}
S(\bm{\tilde{Y}}^{(K)}) = \sum_{j=1}^n \sum_{i=1}^{j} \begin{cases}
\bm{\Phi}^{\rm (ent)}_{i, y_i} & (i = j), \\
\frac{1}{n} {\mathsf \Phi}^{\rm (rel)}_{i,j,\hat{r}} & (i < j),
\end{cases}
\label{eq:score_overall} 
\end{equation}
where $\hat{r}$ is the relation hypothesis at cell $(i,j)$. Thus, candidates can be re-ordered based on both NE and relation scores. We select the candidate yielding the highest overall score as the final prediction. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experiments}
\label{sec:experiments}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Datasets}
\label{sec:dataset}

We assessed the performance of the proposed method for NER and RE using three datasets: CoNLL04 \cite{roth-yih-2004-linear}, ACE05, and Wet Lab Protocol Corpus (WLPC) \cite{kulkarni2018wetlab}. Adhering to the conventional evaluation scheme, we used micro F1-scores for the NER and RE for evaluation. To evaluate NER, we regarded an entity mention as correct if both its label and span region\footnote{ The definition of an entity's span region varied in the dataset. We followed the major evaluation scheme of each dataset for a fair comparison.} were identical to those of the ground truth. In addition, the ability of the proposed model to capture cross-sentence dependencies in NER was evaluated using CoNLL03 and ACE05, which include markers for document boundaries.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{CoNLL04}  This dataset contains four entity types and five relation types collected from news stories. We followed the setup and data split of \citeA{gupta-etal-2016-table} and \citeA{spert}, which are similar to those of \citeA{miwa-sasaki-2014-modeling} and \citeA{zhang-etal-2017-end}. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{ACE05}  We used the English corpus comprising seven coarse-grained entity types and six coarse-grained relation types, collected primarily from news and broadcasts. We followed the data splits, pre-processing method, and task settings of \citeA{li-ji-2014-incremental} and \citeA{miwa-bansal-2016-end}. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{CoNLL03} This dataset contains four different entity types similar to CoNLL04 \cite{roth-yih-2004-linear}, collected from news stories. We used this dataset to measure the performance of the model on the NER task, which considers cross-sentence contexts within a document.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{WLPC} This dataset contains 18 entity types and 13 relation types. WLPC records the rules or instructions for conducting biology or chemistry laboratory experiments, in contrast the datasets mentioned above in terms of the data domain.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Experimental Settings}

The proposed model was implemented in PyTorch \cite{pytorch} with the HuggingFace Transformer package \cite{Wolf2019HuggingFacesTS}, utilizing BERT\textsubscript{BASE} (cased) \cite{devlin-etal-2019-bert} as a pre-trained language model by default. We examined the performance of the proposed model in comparison to that of other pre-trained language models for a fair comparison. All experiments were performed on a single Tesla V100-SXM2-16GB GPU.


During training, the newly introduced parameters followed a learning rate different from those of the pre-trained language models, similar to \citeA{Wadden2019EntityRA}. We applied the AdamW algorithm implemented in PyTorch for parameter updates  \cite{loshchilov2018decoupled}. The major hyperparameters are listed in Table~\ref{tab:hyper}, and the mean values of all the evaluation metrics recorded over five runs for each experiment are reported.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[t]
\input{08table01.tex}
\hangcaption{Hyperparameter settings when using BERT\textsubscript{BASE}. We adopt a learning rate scheduler that scaled the learning rate up linearly from $0$ to the value recorded above in the warm-up period of $0.2 \times \mbox{(total number of epochs)}$, and then scales the learning rate down using a cosine function.}
\label{tab:hyper}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Main Results}
\label{sec:results}

Table~\ref{tab:results} reports the performance of our method on the datasets in comparison with those obtained by several recent studies on joint NER and RE.\footnote{We exclude \citeA{zhong-chen-2021-frustratingly} here since their approach is pipelined.} To evaluate RE, we report the scores computed from two different criteria for a fair comparison with the previous work; \texttt{RE} is indifferent towards incorrect predictions of NE labels, whereas \texttt{RE+} requires the NE labels of relation arguments to be correct.

On the CoNLL04 dataset, the proposed method achieved state-of-the-art performance when utilizing SpanBERT\textsubscript{LARGE} \cite{joshi-etal-2020-spanbert}, showing significant performance improvement in both NER and RE. However, our system failed to improve the performance of \citeA{ji-etal-2020-span} on RE when the pre-trained language models were aligned (72.7 vs.\ 74.3), and the performance of NER was comparable (90.5 vs.\ 90.2). We attribute this difference to the existing method's ability to capture span-specific representations obtained by multi-head attention, which is beyond the scope of the present study.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 2
\begin{table}[t]
\input{08table02.tex}
\hangcaption{Micro-averaged F1-scores on the test sets CoNLL04, ACE05 and WLPC. ``pre-trained LM'' shows the pre-trained language model utilized by each system. Scores from existing work are also reported. }
\label{tab:results}
\vspace{-0.75\Cvs}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

On the ACE05 dataset, when utilizing ALBERT\footnote{We applied \texttt{albert-xxlarge-v1} to be consistent with the model compared.} \cite{Lan2020ALBERT} as the pre-trained language model, the proposed system performed comparably to \citeA{wang-lu-2020-two} for both NER (89.8 vs.\ 89.5) and RE (67.7 vs.\ 67.6). When utilizing BERT\textsubscript{BASE} as the pre-trained language model, the proposed approach outperformed the model used for comparison \cite{Wadden2019EntityRA} in RE (66.1 vs.\ 63.4), whereas the prior method showed a superior performance on NER (88.0 vs.\ 88.6). However, their method received document-level contexts as input, making it incomparable with our model trained only with sentence-level contexts. In addition, aside from the ACE05 dataset, \citeA{Wadden2019EntityRA} trained their model on OntoNotes \cite{pradhan-etal-2012-conll} to capture co-reference information, leading to further incomparability in the sense of the data scale.
However, as may be noted from Table~\ref{tab:ner} of \S~\ref{sec:doclevel}, the proposed method still outperformed \citeA{Wadden2019EntityRA} in terms of NER with document-level context given as the input (89.1 vs.\ 88.6).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[t]
\input{08table03.tex}
\hangcaption{Results of NER task on the CoNLL03 and ACE05 test sets. The values for the existing study are previously reported scores. We include a baseline BERT, following the original study settings, wherein each word is represented by its first subword token.}
\label{tab:ner}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

On the WLPC dataset, although it failed to show state-of-the-art performance on NER (78.4 vs.\ 79.7), the proposed method achieved comparable performance to previously developed models on RE (66.0 vs.\ 65.9). Similar results were observed for the CoNLL04 dataset. When using ALBERT as the pre-trained language model, the performance of the proposed method was inferior to that of an alternative \citeA{wang-lu-2020-two} on NER (89.7 vs.\ 90.1) but performed comparably on RE (73.9 vs.\ 73.8). These results indicate the strength of the proposed method in extracting relations, despite the simplicity of the model architecture. 

Therefore, from the experimental results for CoNLL04, ACE05, and WLPC, we conclude that the proposed method is not only simple but also effective in extracting entities and relations.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Document-Level NER}
\label{sec:doclevel}

As described in \S~\ref{sec:results}, the proposed method did not outperform \citeA{Wadden2019EntityRA} on ACE05 NER owing to the inaccessibility of cross-sentence information such as co-references. This subsection describes how we addressed the performance gap by merely providing multiple sentences into the model without modifying the model architecture. Specifically, we split a document into segments comprising multiple sentences such that each segment was not longer than 256 subword tokens. We introduced this length restriction owing to the limitation in the memory capacity of the GPU. Assuming each segment to be a sequence of subword tokens from multiple sentences, we fed each segment to the model with multiple sentences separated by token {\tt [SEP]}, similar to \citeA{devlin-etal-2019-bert}.

Table~\ref{tab:ner} presents performance of the NER models on CoNLL03 and ACE05 test sets with and without document-level inputs. We used ``BERT (our implementation)'' as the baseline, and then equipped the BERT encoder with extra modules, as described in \S~\ref{sec:ner}.
We observed that document-level inputs aided to improve the performance of our model on both datasets; therefore, our model outperformed other models, including \citeA{Wadden2019EntityRA} and BERT \cite{devlin-etal-2019-bert}.
By comparing the prediction results, we can conclude that document-level inputs were able to improve the prediction performance for multiple occurrences of the same entity by gathering contexts from different occurrences. Specific pieces of evidence with examples are listed in Table~\ref{tab:coref_examples}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 4
\begin{table}[b]
\input{08table04.tex}
\hangcaption{Typical examples wherein the proposed NER model showed improvements with document-level inputs. A \textbf{boldface} word denotes the target for predicting the NE label, and an \textit{italic} word denotes a co-reference to the boldface word from another sentence. The NE labels shown on top of each example are the predictions from the sentence-level input (Sent.), document-level input (Doc.), and ground-truth label (Gt.). We can infer that the model with the document-level input made the correct predictions by looking at the coreferential words.}
\label{tab:coref_examples}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Although this technique cannot be directly applied to RE because the table size is $\mathcal{O}(n^2)$, we tested our RE model in a pipelined fashion using predictions of NER with document-level inputs on the ACE05 test set. However, no performance boost was observed for the RE. By analyzing the predicted RE instances, we discovered that document-level NER increased the overall performance by capturing co-references. However, it failed to correctly label some of the NEs essential for RE. This observation emphasizes the importance of improving the design of the RE model and of developing a better approach to combining sentence- and document-level context during RE. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Simplicity of Proposed Method}
\label{sec:simplicity}

We have reported the primary results of experiments evaluating the efficacy of the proposed method, from which we observed that the method achieved high performance on multiple benchmarks. Our model is simple in architecture based on pre-trained BERT encoders, adding a minimal number of task-specific modules. In this subsection, we quantitatively show the simplicity of the proposed model from the perspectives of both computational space and time requirements. 
As for computational space, our model introduces a relatively small number of trainable parameters, thus reducing memory consumption during both training and prediction. Specifically, we compared the number of newly introduced parameters in our system with that of several typical existing methods. We chose \citeA{Wadden2019EntityRA}\footnote{\url{https://github.com/dwadden/dygiepp}} as a representative span enumeration system and \citeA{wang-lu-2020-two}\footnote{\url{https://github.com/LorrinWWW/two-are-better-than-one}} as a non-span enumeration system. As shown in Table~\ref{tab:trainable_params}, the memory consumption of our system was approximately 1/3 that of the methods compared. Based on these results, we may conclude that compared with existing systems, the proposed system introduces fewer trainable parameters while achieving comparable or better performance.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 5
\begin{table}[b]
\input{08table05.tex}
\hangcaption{Comparison of the number of trainable parameters excluding those from pre-trained LMs. In the column ``Pre-trained LM'', the number in each parenthesis denotes the dimension of token embedding (that is, the dimension of the output) of the corresponding pre-trained LMs.}
\label{tab:trainable_params}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

With respect to computational time, our model simplifies the decoding process by predicting off-diagonal cells simultaneously, thus improving prediction speed. We chose \citeA{wang-lu-2020-two} for comparison because it is the latest system that applies the table-filling strategy \cite{miwa-sasaki-2014-modeling}. As shown in Table~\ref{tab:decoding_speed}, when the pre-trained LM was aligned, our model decoded approximately three times faster than the methods compared using a greedy search (that is, beam size was 1) on a single Tesla V100-SXM2-16GB GPU, while showing a comparable performance. Even when the beam size increased to 5, our model decoded approximately twice as fast as the competitor, showing a clear advantage in decoding speed. These results indicate that compared with \citeA{wang-lu-2020-two}, the latest table-filling system, our system decodes faster while performing comparably.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 6
\begin{table}[t]
\input{08table06.tex}
\hangcaption{Comparison of the average decoding speed on the CoNLL04 test set. The pre-trained LM was aligned with ALBERT\textsubscript{XXLARGE} to ensure a fair comparison.}
\label{tab:decoding_speed}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Apart from quantitative measurements, the simplicity of our method may also be noted in its straightforward design. Compared to previous work, our method involves neither negative sampling \cite{spert,ji-etal-2020-span} nor graph propagation \cite{luan-etal-2019-general,Wadden2019EntityRA,qian-etal-2019-graphie}, resulting in a straightforward and stable training process.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Analyses}
\label{sec:analyses}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Decoding Order}
\label{sec:dec_order}

Existing methods for the joint extraction of entities and relations using the table-filling approach rely on history-based decoding, which involves filling up the lower (or upper) triangular part of a table cell-by-cell in a predefined order \cite{miwa-sasaki-2014-modeling,gupta-etal-2016-table,zhang-etal-2017-end}.
These methods assume that previous decisions help in subsequent decisions, which may involve long-range dependencies.
In contrast, our model does not use the decoding history for relation labels, but focuses on predicting them all at once, using the tensor dot product.

A natural question arises as to whether history-based decoding remains useful for the proposed model.
To find a solution, we designed a variant of the RE model to predict relation labels cell by cell. During both training and prediction, the model explicitly considers three pieces of information: (1) the relation label of the previous timestep, (2) the relation label at the cell to the left of the target cell, and (3) the relation label at the cell below the target label.
Specifically, we modified Equation~\ref{eq:hrel} to utilize embeddings of the relation labels at $(i,j-1)$, $(i+1,j)$, and the previous timestep when deciding the relation label for the element $(i,j)$. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 5
\begin{figure}[t]
\begin{center}
\includegraphics{29-1ia8f5.pdf}
%%%%\label{fig:order_0}}
%%%%\label{fig:order_1}
%%%%\label{fig:order_2}}
%%%%\label{fig:order_3}
\end{center}
\hangcaption{Static decoding orders pre-defined in this section. The captions of each sub-figure describe the general decoding strategy, while the numbers in the cells indicate the specific decoding order.}
\label{fig:order}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We defined four decoding orders, as shown in Figure~\ref{fig:order}. Our design incorporated two types of label dependencies: inter-sub-diagonal dependencies introduced by utilizing labels below and to the left of the target cell, and intra-sub-diagonal dependencies introduced by utilizing labels predicted at the previous timestep. 
%%%%Taking Figure~\ref{fig:order_0} as an example, 
    Taking Figure~\ref{fig:order}(a) as an example, 
when deciding a relation label for the element $(2,5)$, inter-sub-diagonal dependencies were covered by relation labels at $(2,4)$ and $(3,5)$, whereas the intra-sub-diagonal dependency was covered by a relation label at $(1,4)$. We examined the influence of inter-and intra-sub-diagonal dependencies separately via an ablation test. Specifically, to remove intra-sub-diagonal dependencies, we excluded the embedding of relation label at previous timestep from feature representation; to remove inter-sub-diagonal dependencies, we excluded embeddings of relation labels below and to the left of a target cell from the feature representation.


Experimental results on the CoNLL04 test set are shown in Table~\ref{tab:neighbor}. By incorporating historical information, the proposed model exhibited a slight performance gain in terms of RE, while the removal of either intra-sub-diagonal or inter-sub-diagonal information lowered the performance of the RE to a small extent.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 7
\begin{table}[t]
\input{08table07.tex}
\hangcaption{Performance of the proposed model on the CoNLL04 test set with history-based decodings. ``Once'' stands for the proposed model that predicts all the relation labels at once. ``- Intra.'' represents a model variant that excludes intra-sub-diagonal information. ``- Inter.'' represents a model variant that excludes the inter-sub-diagonal information. For simplicity, only the results when the beam size=1 (that is, greedy search) are reported.}
\label{tab:neighbor}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

However, compared with \citeA{miwa-sasaki-2014-modeling}, where the worst and best orders were significantly different in terms of performance, the improvement of our system was subtle. It is difficult to identify the reason for the experimental results; however, the RE model might utilize long-range dependencies from the BERT encoder to make decisions.
In addition, history-based decoding increases the number of parameters and the complexity of label predictions by introducing extra parameters for relation embeddings and additional classifiers. The experimental results suggest that predicting the relation labels at once is sufficient for the model proposed.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{NER's Effect on RE}
\label{sec:ner2re}

Previous studies have reported that joint training with NER 
\pagebreak
allows the modeling of cross-task dependencies, whether implicitly or explicitly, which helps improve RE performance \cite{miwa-bansal-2016-end,zhang-etal-2017-end,lin-etal-2020-joint}. We analyzed how RE benefits from joint training with NER under our design by examining the performance of RE when NER predictions are disturbed.  


As introduced in \S~\ref{sec:ner}, the labeling of NEs follows a BILOU notation and produces labels containing both segment and category information for each word. For example, ``Johanson'' in Figure~\ref{fig:example} is labeled as \texttt{B-PER}, where \texttt{B} represents the segment information and \texttt{PER} presents the category information. Therefore, we refer to prefixes such as \texttt{B} as segment indicators and tags such as \texttt{PER} as category indicators. To disturb NER predictions, we manipulated NE labels to prevent segment information or category information from being correctly passed down to the RE model. Specifically, to hide category information, we reduced all the NE categories to an abstract entity category ``X''. To hide segment information, we replaced the segment indicator with a randomized counterpart. Examples of these disturbances for sentence ``Mrs. Tsutayama is from Kumamoto Prefecture in Japan'' are shown in Figure~\ref{fig:joint}. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 6
\begin{figure}[b]
\begin{center}
\includegraphics{29-1ia8f6.pdf}
\end{center}
\caption{Example of hiding segment and category information by manipulating the NE labels.}
\label{fig:joint}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Using several distinct settings, we evaluated the effects of disturbances on RE performance. The results are reported in Table~\ref{tab:confuse}, based on which we observe that disturbances of category information during both training and prediction did not significantly affect the RE performance (Settings 2 and 3). We attribute this to the use of pre-trained language model encoders, from which NE labels can be easily predicted from word embeddings to a certain extent. Similar findings have been reported in \citeA{luo-etal-2020-named}, wherein an NER model using only pre-trained word embeddings achieved remarkable results. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 8
\begin{table}[t]
\input{08table08.tex}
\hangcaption{Performance of the proposed RE model on the CoNLL04 test set when disturbing the NER labels. ``Seg.'' is short for ``segment'' and ``Cat.'' is short for ``category.'' \ding{51} indicates that the corresponding information is provided for training, or the corresponding predicted results are used for prediction. \ding{55} indicates that the corresponding information was discarded both during training and prediction. \ding{66} indicates that the corresponding information was disturbed by randomization. \ding{72} indicates that the corresponding ground-truth information was provided.}
\label{tab:confuse}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

In contrast, we observed a performance decrease when discarding segment information (Setting 4). Disturbing segment information during prediction led to an even more significant performance drop, similar to Setting 5. However, once provided with ground-truth segment indicators, the RE model exhibited significant performance improvements even when the NER model was completely untrained (Setting 6). These results indicate the importance of correct segment information for relation predictions. 

Furthermore, if jointly trained with the NER model and provided with ground-truth NE labels, the performance of the RE model increased again, as shown in Settings 7 and 8. This improvement in performance can be attributed to improvements in word embeddings refined by joint training with the NER model.

From the analyses above, we conclude that joint learning of RE with NER aids to improve the embeddings under the proposed architecture, which is the major effect of NER on RE.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Effect of Beam Search}
\label{sec:beam}

As described in \S~\ref{sec:training}, we replaced the greedy search with a beam search during prediction to enable interactions between NER and RE. Our model selects the best prediction among $K$ candidates using balanced overall scores computed from the NE and relation scores (Equation~\ref{eq:score_overall}). In this section, we report the effect of using this beam search strategy on the proposed system.

We first examined how the performance of the NER and RE models varied with beam size on CoNLL04 and ACE05, based on the experimental results shown in Figure~\ref{fig:beam}. We plotted the results using BERT\textsubscript{LARGE} as the pre-trained language model. Table~\ref{tab:beam} details the performance of the variants built on the other pretrained models. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 7
\begin{figure}[b]
\begin{center}
\includegraphics{29-1ia8f7.pdf}
%%%%\label{fig:beam_conll04_ner}}
%%%%\label{fig:beam_ace05_ner}
%%%%\label{fig:beam_conll04_re}}
%%%%\label{fig:beam_ace05_re}
\end{center}
\hangcaption{Performance of the proposed RE model on CoNLL04 and ACE05 test sets when decoded jointly with NER model using beam search with an increasing beam size. The pre-trained language model was BERT\textsubscript{LARGE}.}
\label{fig:beam}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

From Figure~\ref{fig:beam}, it may be concluded that when the beam size was small, our models obtained a performance gain on both the CoNLL04 and ACE05 datasets in terms of the F1 score. However, after reaching a certain boundary, a wider beam size no longer contributed to the performance improvement. Similar tendencies were observed when utilizing other pre-trained models, as illustrated in Table~\ref{tab:beam}.
These results show that beam search with a balanced overall score computed from NER and RE scores aids to improve the prediction accuracy, which further indicates that the joint prediction of NER and RE is mutually beneficial to each other under our architecture. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 9
\begin{table}[t]
\input{08table09.tex}
\hangcaption{Performance of the proposed RE model on the CoNLL04 test set when decoded jointly with the NER model using beam search with an increasing beam size.}
\label{tab:beam}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

In addition, although they have similar tendencies, the curves of CoNLL04 and ACE05 differ from one another. When increasing the beam size, the F1 score of NER on the CoNLL04 test set improved by approximately 0.5, 
%%%%as shown in Figure~\ref{fig:beam_conll04_ner}, 
    as shown in Figure~\ref{fig:beam}(a), 
while the improvement on the ACE05 dataset was significantly less. 
%%%%This dissimilarity can be observed in Figures ~\ref{fig:beam_conll04_re} and~\ref{fig:beam_ace05_re}. 
    This dissimilarity can be observed in Figures ~\ref{fig:beam}(c) and~\ref{fig:beam}(d). 
From these comparisons, we conclude that the effect of beam search is not always notable and depends on the nature of the corpus on which the system is trained.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Effect of Weighted Loss}

During multi-task training, various existing systems have adopted weighted losses to balance their effects among multiple tasks \cite{li-etal-2019-entity,Wadden2019EntityRA,ji-etal-2020-span}. Specifically, instead of directly summing up the objective of each task, similar to Equation~\ref{eq:loss}, hyperparameters $\lambda^{\rm (ent)}$ and $\lambda^{\rm (rel)}$ are introduced to assign weights corresponding to the objectives. To explore whether the modification benefits the training process, we trained our model with weighted losses and examined its performance. 

Specifically, we varied $\lambda^{\rm (ent)}$ from 0.1 to 0.9, while retaining $\lambda^{\rm (ent)} + \lambda^{\rm (rel)} = 1$. The F1-scores of the predicted results for the CoNLL04 test set are shown in Table~\ref{tab:lambda}. Although adjusting the weights did not lead to a dramatic change in model performance, we observed that, in general, when increasing $\lambda^{\rm (ent)}$, the F1 score of NER improved, and the F1 score of RE decreased. When focusing on the metric \texttt{RE+}, which requires not only correct relation labels but also correct NE labels, we observed better performance when $\lambda^{\rm (rel)}$ was relatively large. 

Alternatively, in addition to setting $\lambda^{\rm (ent)}$ and $\lambda^{\rm (rel)}$ as hyperparameters, we also attempted to use the normalization defined in \S~\ref{sec:training}. Under this setting, $\mathcal{L}^{\rm (rel)}$ is ``normalized'' similar to Equations~\ref{eq:re_normalization} and \ref{eq:score_overall} before back-propagation. Consequently, the operation downscaled the influence of $\mathcal{L}^{\rm (rel)}$ during the training process. As shown in Table~\ref{tab:norm}, compared with simply summing up $\mathcal{L}^{\rm (ent)}$ and $\mathcal{L}^{\rm (rel)}$, adding normalization worsened the performance of RE.

As shown in both Tables~\ref{tab:lambda} and~\ref{tab:norm}, we observed performance drops when downscaling $\mathcal{L}^{\rm (rel)}$. This finding is similar to that of \citeA{Wadden2019EntityRA}, where $\lambda^{\rm (ent)}$ was set to 0.2, and $\lambda^{\rm (rel)}$ to 1.0. A possible explanation is that because RE tasks are more complex than NER, it is potentially beneficial to have $\lambda^{\rm (rel)} > \lambda^{\rm (ent)}$ during training. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 10 and 11
\begin{table}[t]
\setlength{\captionwidth}{200pt}
\begin{minipage}[t]{200pt}
\input{08table10.tex}
\hangcaption{Performance of the proposed model on the CoNLL04 test set when trained with a weighted sum of cross-entropy losses of NER ($\mathcal{L}^{\rm (ent)}$) and RE ($\mathcal{L}^{\rm (rel)}$).}
\label{tab:lambda}
\end{minipage}
\hfill
\raisebox{45.5pt}[0pt][0pt]{%
\begin{minipage}[t]{200pt}
\input{08table11.tex}
\hangcaption{Performance of the proposed model on the CoNLL04 test set when trained with and without normalization of $\mathcal{L}^{\rm (rel)}$.}
\label{tab:norm}
\end{minipage}
}%
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Ablation Test}

To better understand the significance of our proposed method, we conducted ablation tests. Features were gradually removed from the model, and the consequent performance drops were measured.
Specifically, we ablated the features of the label embedding $\bm{l}_{y_{i-1}}$ and the previous span $f(\bm{e}_{w_{{\rm first}(i-1)}}, \cdots, \bm{e}_{w_{i-1}})$ from Equation \ref{eq:entity_repr}.
As can be seen from Table \ref{ablation}, the removal of the features of previous spans from the model had the highest negative impact, whereas the removal of label embeddings at previous time steps had a relatively insignificant negative impact. This result validates our assumption that span-level features are beneficial for representing entities in both NER and RE.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 12
\begin{table}[t]
\input{08table12.tex}
\hangcaption{Ablation test of features used in this study on the CoNLL04 test set. ``- Label'' presents the performance when the label embeddings at previous timesteps are removed from the model. ``- Span'' removes the features of the previous span. ``- Both'' presents the results when both label embeddings at previous timesteps and features of the previous span are removed.}
\label{ablation}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Error Inspection}
\label{sec:err}

This section contains descriptions of incorrect predictions from the proposed method to delineate future directions for improvement. Among the predicted results of the proposed system on the CoNLL04 test set, we randomly sampled 100 error instances and categorized them into several predefined error categories. For NER, we categorized the instances by error type. Here, we regarded an error as a false positive only when the predicted span did not overlap with any ground-truth entity span. Similarly, we regarded an error as a false negative only when the model failed to detect any subset of a ground-truth entity span. For RE, we categorized the instances by error causes because the distribution of error types was uninformative (that is, most of the errors were false negatives). The distribution and examples of the remaining errors are shown in Figure~\ref{fig:err} and Table~\ref{tab:err}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 8
\begin{figure}[t]
\begin{center}
\includegraphics{29-1ia8f8.pdf}
%%%%\label{fig:error_ner}}
%%%%\label{fig:error_re}
\end{center}
\caption{Distribution of remaining error types (causes) on the CoNLL04 test set.}
\label{fig:err}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Incorrect NE Span} Cases where the NER model predicted slightly incorrect spans. Typical errors in this category involve adding/missing a nearby phrase in an NE span. Table~\ref{tab:err}(a) is an example where the phrase ``on Earth'' can be interpreted as a prepositional phrase or part of a proper noun.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Incorrect NE Type} Cases where the NER model predicted incorrect NE labels for entity mentions. These cases usually occur when an entity is using different NE types. Table~\ref{tab:err}(b) illustrates that ``Charing Cross Hospital'' can be categorized as \textsc{Organization} if we consider the NE alone, but it was actually annotated as \textsc{Location} in the context (indicating the location of the event `died').


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Lack of Knowledge} Cases where the RE model failed to recognize implicit relations. In Table~\ref{tab:err}(c), it is not easy to recognize the relation instance $\mathrm{(\text{Livingston}_{Loc}, LocatedIn, \text{Montana}_{Loc})}$ from only the sentence without the knowledge of entities ``Livingston'' and ``Montana''. Fortunately, the RE model managed to predict the relation instance correctly in this example. However, it is even more difficult to infer the relation instance $\mathrm{(\text{Livingston}_{Loc}, LocatedIn, \text{Rocky Mountains}_{Loc})}$ from the text; the inclusion relation between ``Rocky Mountain'' and ``Livingston'' cannot be clearly understood without the knowledge of entities.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Propagated Error} Cases where the RE model failed to predict the relation correctly due to NE label/span error propagated from the NER model. Table~\ref{tab:err}(d) shows that the relation between ``Darryl Breniser'' and ``Blue Ball'' was misclassified as \textsc{WorkFor} because ``Blue Ball'' is mistaken as an \textsc{Organization} entity in the NER model. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Lack of Global Constraints} Cases where the RE model could have avoided mistakes with global constraints or dependencies. Typical errors in this category involve, as shown in Table~\ref{tab:err}(e), inferring that the same person lives in two different places (Soviet Union and China). Therefore, the proposed method cannot explicitly consider associations between relation predictions because the relation labels are predicted independently of one another.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 13
\begin{table}[p]
\input{08table13.tex}
\hangcaption{Typical error cases of the proposed method on the CoNLL04 test set. Cases \textbf{(a)} and \textbf{(b)} represent the errors caused by the NER model, and cases \textbf{(c)--(f)} represent the errors caused by the RE model. Each RE case shows a sentence with the NE labels in the first line, followed by the relation tuples of the ground-truth annotation and prediction.} 
\label{tab:err}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Lack of Syntactic Information} Cases where the RE model could have avoided mistakes with the assistance of syntactic information. As shown in Table~\ref{tab:err}(f), the model incorrectly predicted an RE instance $\mathrm{(\text{Gerald Baliles}_{Peop}, LiveIn, \text{New Hampshire}_{Loc})}$. The error can potentially be eliminated by utilizing syntactic structures since ``New Hampshire'' is a phrase from the directed speech(that is, inside the quotation marks), and ``Gerald Baliles'' is a phrase from the reporting clause (that is, outside the quotation marks).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Discussion}

As introduced in \S~\ref{sec:intro}, two of the major limitations of table filling-based approaches \cite{miwa-sasaki-2014-modeling,gupta-etal-2016-table,zhang-etal-2017-end} lie in the necessity of feature engineering and exploration of the search order. This study focuses on an investigation into how contextualized representations assist in overcoming these limitations and improving table-filling approaches. We modeled NER as a sequential labeling task to maintain computational efficiency and that of RE as independent pair-wise classifications based on the dot-product mechanism to reduce the model's complexity. 

As shown in \S~\ref{sec:results}, the utilization of BERT-based contextualized representations improved the performance of both NER and RE because the expressiveness of pre-trained embeddings positively influenced the final evaluation score. We attribute the performance improvement to the ability of the contextualized representation to capture long-range dependencies. 

Two key facts support the positive influence of contextualized representations. Firstly, according to \S~\ref{sec:doclevel}, expanding the input from the sentence level to the document level has been shown to effectively improve performance, enabling models to capture cross-sentence dependencies such as co-references. Secondly, according to \S~\ref{sec:dec_order}, the improvement gained by incorporating the decoding order was subtle compared with existing work \cite{miwa-sasaki-2014-modeling}. One possible explanation is that the historical information previously employed by the decoding order may have already been encoded into contextualized representations. 


Another research question concerns the effect of joint learning in the proposed architecture. According to \S~\ref{sec:ner2re}, we observe that joint training with NER improved the span-level representations, thus further benefiting RE. 

The contributions of this study can be highlighted in both industrial and research fields. From the perspective of industry, we have developed and revealed a practical tool for extracting entities and relations from plain text. The model can be trained to extract essential information from real-world contexts with good efficiency and accuracy. Furthermore, the proposed model is language-agnostic and generic, as well as convenient for adapting to tasks in other languages or other natural language analysis tasks, provided with corresponding pre-trained language models. Notably, while being language-agnostic is one of the properties of our system, it is not an advantage over prior methods. Compared with existing language-agnostic systems \cite{miwa-sasaki-2014-modeling,luan-etal-2019-general,Wadden2019EntityRA,lin-etal-2020-joint,wang-lu-2020-two}, the advantage of the proposed system is its simplicity during both training and prediction, as described in \S~\ref{sec:simplicity}. For this reason, our system can act as a simple but effective tool for entity and relation extraction.

From the perspective of further research, we hope that the proposed system can serve as a strong baseline for this subject area. We also conducted error inspections as in \S~\ref{sec:err}, suggesting possible directions to investigate when considering further improvement of entity and relation extraction systems. Moreover, we have shown that simple mechanisms, such as dot products, are reasonable for modeling pair-wise relationships on top of transformers. We hope that this finding can influence future research efforts to re-consider approaches to address various tasks that can be formulated as relation extraction processes.


However, the proposed method nonetheless involves several limitations. First, the interaction between the NER and RE is not explicitly modeled. Both training and prediction follow a pipelined flow, where the NER and RE models share only the parameters of the pre-trained encoder. Second, the system ignores the dependencies between RE pairs. Relations between entity pairs are independent of one another, which may negatively impact performance (\S~\ref{sec:err}).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}
\label{sec:conclusion}

In this study, we have presented a method for the joint extraction of NE and relations based on table representation, using contextualized word embeddings to represent entity mentions. We applied a tensor dot product to predict the relation labels simultaneously. During prediction, we defined an overall score function computed from the NE and relation scores and decided the final predictions in a joint manner. 

We have evaluated the proposed system on the CoNLL04, ACE05, and WLPC datasets. The experimental results show that the proposed method achieved state-of-the-art performance, exhibiting a significant improvement in RE. Furthermore, when the experimental settings were aligned with those of the existing methods, the proposed systems still outperformed their competitors, indicating the superiority of our model's architecture. By changing the input unit from sentence level to document level, the proposed NER model exhibited the ability to capture cross-sentence dependencies.

To analyze the performance of the proposed approach, we examined the effects of decoding order. We trained model variants on four predefined static decoding orders and confirmed that predicting the relation labels all-at-once is sufficient for the proposed method. We experimentally demonstrated that, for the proposed RE model, joint training with NER improves word embeddings. By exploring the effect of the beam search, we found that the joint prediction for NER and RE is mutually beneficial. 

In the future, we plan to explore an approach incorporating global constraints in the RE model, which currently predicts all relation labels independently. It may also be worthwhile to consider an alternative method to combine document- and sentence-level information during RE.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment

This paper is based on results obtained from a project, JPNP18002, commissioned by the New Energy and Industrial Technology Development Organization (NEDO).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.6}
\bibliography{08refs}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Youmi Ma}{Ph.D. student at Tokyo Institute of Technology. She received the bachelor's degree (Bachelor of Engineering) from ShanghaiTech University, Shanghai, China in 2019 and the M.S. degree (Master of Engineering) from Tokyo Institute of Technology in 2021.}

\bioauthor[:]{Tatsuya Hiraoka}{Ph.D. student at Tokyo Institute of Technology. He received the bachelor's degree (Bachelor of Arts) from Waseda University in 2017 and the M.S. degree (Master of Engineering) from Nara Institute of Science and Technology in 2019.}

\bioauthor[:]{Naoaki Okazaki}{Professor at Department of Computer Science, School of Computing, Tokyo Institute of Technology. He obtained his Ph.D. from University of Tokyo in 2007. He worked as a post-doctoral researcher in University of Tokyo (in 2007--2011) and as an associate professor in Tohoku University (in 2011--2017). He is also a visiting research scholar of the Artificial Intelligence Research Center (AIRC), AIST. }
\end{biography}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\biodate




\end{document}

