\documentclass[english]{jnlp_3.7}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{hangcaption_jnlp}
\usepackage{array}
\allowdisplaybreaks


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition

%%%%%\documentclass[english]{jnlp_3.7}
%%%%%\usepackage{jnlpbbl_2.2}
%%%%%\usepackage[dvipdfmx]{graphicx}
%%%%%\usepackage{amsmath}
%%%%%\usepackage{url}
%%%%%\usepackage{hangcaption_jnlp}
%%%%%\usepackage{array}
%%%%%\allowdisplaybreaks

\usepackage{bm} %ynaga
%%%%%\renewcommand{\UrlFont}{\ttfamily\small}

%%%%%\usepackage{algorithm, amsfonts, amsthm, mleftright, multirow, booktabs} % amsma
%%%%%\usepackage{siunitx}
\usepackage{algorithm}
\usepackage{booktabs}

%%%%%\renewcommand{\figurename}{Figure}
%%%%%\renewcommand{\tablename}{Table}

%%%%%\theoremstyle{definition}
%%%%%\newtheorem{defi}{Def.\null}
%%%%%\newcommand{\argmax}{\mathop{\rm argmax}\limits}
%%%%%\newcommand{\argmin}{\mathop{\rm argmin}\limits}
%%%%%\newcommand{\argmaxinline}{{\rm argmax}\limits}
%%%%%\newcommand{\ave}{\mathop{\rm ave}\limits}
%%%%%\newcommand{\softmax}{\mathop{\rm softmax}\limits}
%%%%%\newcommand{\1}{\mbox{1}\hspace{-0.25em}\mbox{l}}

\usepackage{algpseudocode}
%%%%%\usepackage{comment}
\usepackage{color}

%%%%%\iffalse
%%%%%\renewcommand\Authands{\quad}
%%%%%\renewcommand\Authsep{\quad}
%%%%%\author[$\dag\ddag$]{\textbf{Kosuke Nishida}}
%%%%%\author[$\S$]{\textbf{Naoki Yoshinaga}}
%%%%%\author[$\dag$]{\textbf{Kyosuke Nishida}}
%%%%%\affil[$\dag$]{NTT Human Informatics Laboratories, NTT Corporation}
%%%%%\affil[$\ddag$]{The University of Tokyo}
%%%%%\affil[$\S$]{Institute of Industrial Science,
%%%%%	University of Tokyo, Tokyo}
%%%%%\affil[$\dag$]{\texttt{\{kosuke.nishida.ap, kyosuke.nishida.rx\}@hco.ntt.co.jp}}
%%%%%\affil[$\S$]{\texttt{ynaga@iis.u-tokyo.ac.jp}}
%%%%%\fi

\newcommand{\yn}[1]{\textcolor{black}{#1}}
%%%%%\newcommand{\new}[1]{\textcolor{black}{#1}}

%参照なしのfootnote
%%%%%\usepackage{lipsum}
%%%%%\newcommand\blfootnote[1]{%
%%%%%	\begingroup
%%%%%	\renewcommand\thefootnote{}\footnote{#1}%
%%%%%	\addtocounter{footnote}{-1}%
%%%%%	\endgroup
%%%%%}

%%%\makeatletter
%%%\renewcommand{\algocf@capseparator}{~~}%
%%%\makeatother

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{31}
\Number{2}
\Month{June}
\Year{2024}
%%%%\Issuetitle{「　　　　」特集号}


\received{2023}{11}{1}
\revised{2023}{12}{28}
\accepted{2024}{1}{22}



\setcounter{page}{407}


%%%%   Title, Author, etc.            %%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Self-Adaptive Named Entity Recognition by Retrieving Unstructured Knowledge}
\eauthor{Kosuke Nishida\affiref{NTT}\affiref{UT} \and Naoki Yoshinaga\affiref{IIS} \and Kyosuke Nishida\affiref{NTT}} 
\eabstract{%
Although named entity recognition ({NER}) assists in extracting domain-specific entities from text (\textit{e.g.}, artists in the music domain), it is expensive to create a large amount of training data or structured knowledge base to perform accurate {NER} in the target domain. Here, we propose a self-adaptive {NER} that retrieves external knowledge from unstructured text to learn the usage of entities that have not been learned well. To retrieve useful knowledge for {NER}, we designed an effective two-stage model that retrieved unstructured knowledge using uncertain entities as queries. Our model predicts the entities in the input and then identifies entities whose predictions are not confident. It then retrieves knowledge by using these uncertain entities as queries and concatenates the retrieved text with the original input to revise the prediction. Experiments on CrossNER datasets demonstrated that our model outperforms 
strong baselines using 2.35 points in the F$_1$-metric.
\textcolor{black}{
We confirmed that knowledge retrieval is important for the NER task and that retrieval based on prediction confidence is particularly useful when the model has long-tail entity knowledge through pre-training.}
}

\ekeywords{Named Entity Recognition, Retrieval-Augmented Language Model, Domain Adaptation}

%%%%%\headauthor{
%%%%%\parbox[t]{0.5\linewidth}{
%%%%%\begin{flushleft}
%%%%%Kosuke Nishida, Naoki Yoshinaga, Kyosuke Nishida
%%%%%\end{flushleft}
%%%%%}}
%%%%%\headtitle{
%%%%%\parbox[t]{0.5\linewidth}{
%%%%%\begin{flushright}
%%%%%Self-Adaptive Named Entity Recognition \\ by Retrieving Unstructured Knowledge
%%%%%\end{flushright}
%%%%%}}

\headauthor{Nishida et al.}
\headtitle{Self-Adaptive Named Entity Recognition by Retrieving Unstructured Knowledge}



%%%%     Affiliation     %%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\affilabel{NTT}{}{NTT Human Informatics Laboratories, NTT Corporation}
\affilabel{UT}{}{The University of Tokyo}
\affilabel{IIS}{}{Institute of Industrial Science, the University of Tokyo}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S1
\section{Introduction}
\renewcommand{\thefootnote}{}
\footnote[0]{Parts of this paper are published at EACL2023 and arXiv22.10}
\renewcommand{\thefootnote}{\arabic{footnote}}
Named entity recognition ({NER}) assists in extracting entities from text
\yn{in various domains} such as
biomedicine \cite{ner_gene}, diseases  \cite{ner_disease}, and COVID-19 \cite{ner_covid19}. 
However, accurate neural
{NER} requires large amounts of training data \cite{ner_data3,ner_data1,ner_data4}.
Additionally, the annotation of a domain-specific {NER} dataset is expensive because it requires the involvement of domain experts.

To overcome the limitations of large training datasets,
compensate for the lack of training data in {NER}, \yn{researchers have utilized} external knowledge.
Traditional feature-based {NER} 
uses \yn{features based on} gazetteers or name lists \cite{gazetteer4,gazetteer3,gazetteers2} as the external knowledge.
Although recent neural
{NER} methods can benefit from 
gazetteers and name lists \cite{gazetteer_nn3,gazetteer_nn1,gazetteer_nn2}, only a few domains with structured knowledge bases (gazetteers) exhibit this advantage. 
Therefore, several studies have employed
raw text (unstructured knowledge) \yn{to perform}
weakly supervised learning 
for general domain-structured knowledge \cite{ner_weak,gazetteer_nn2,ner_bert,ner_kazama}.

In this study, we explored the potential of utilizing unstructured knowledge in the {NER} task by referring to it 
at inference time. 
Our idea was inspired by recent retrieval-augmented language models (LMs) \cite{realm}.
These models are pre-trained with a retrieval-augmented masked language model (MLM) such that they can perform well in open-domain question answering (ODQA) by retrieving relevant unstructured knowledge using a question as a query.
However, as we will later confirm in the experiments, the improvement in the performance of the models designed for ODQA is limited to the {NER} task because it requires an understanding of numerous entities in the input text.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F1
\begin{figure}[b]
\begin{center}
\includegraphics{31-2ia4f1.pdf}
\end{center}
\hangcaption{Concept of self-adaptive {NER}: \yn{the model predicts entity candidates to conduct entity-level retrieval from the unstructured KB\@; then it revises the prediction with reference to the retrieved knowledge.}}
\label{fig:top}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



To address this problem, we propose a retrieval-augmented model capable of determining the entities to prioritize in the input text for knowledge retrieval. 
The proposed \textbf{self-adaptive {NER} ({SA-NER}) with an unstructured knowledge model}, shown in Figure \ref{fig:top}, searches for an unstructured knowledge base ({UKB}) when it lacks confidence in its prediction. 
We created the {UKB} automatically by splitting a raw text corpus into pieces and assigning dense vectors as keys to individual pieces of unstructured knowledge.
To understand local semantics, we designed a retrieval system tailored for {NER}, which predicts entities and then retrieves knowledge in terms of those that are not confident in predicting.


To evaluate the capability of our method for retrieving 
\yn{useful}
knowledge of entities, we conducted experiments on {NER} \textcolor{black}{datasets} in low-resource domains \cite{crossner,finance_ner}, which 
\yn{have domain-specific entities or domain-specific 
types.}

\textcolor{black}{
Following the standard setting of retrieval-augmented LMs, we conducted pre-training on retrieval augmentation in the NER task.
Here, we provide two pre-training settings: pre-trained on a kilo-scale NER dataset and a million-scale Wikipedia-based weakly supervised corpus. We clarified the effect of self-adaptation when the pre-trained model had only retrieval-augmented NER capability and long-tail entity knowledge.}

Our contributions can be summarized as follows:
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{itemize}
\item We are the first to integrate retrieval-augmentation into {NER}. {SA-NER} retrieves entity-level knowledge dynamically for {NER} (\S \ref{sec:model}).
\item In experiments, {SA-NER} outperformed strong baselines pre-trained in a supervised and self-supervised fashion by 1.22 to 2.35 points.
\item We revealed why knowledge retrieval is useful for {NER}. 
We found that our model is effective
on entities whose knowledge is not stored in the model.
\item \textcolor{black}{We confirmed that the knowledge retrieval is crucial in \textcolor{black}{both the kilo- and million-scale settings}, and the filtering of the retrieved knowledge based on the confidence is useful in the million-scale pre-training settings.}
\end{itemize}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2
\section{Task Settings}
As an attempt, we developed {SA-NER} %.
\yn{to solve the problems of {NER} 
using unstructured knowledge.}
{NER} is a sequence-tagging task in which the model inputs a token sequence $X \in V^L$, where $V$ is the vocabulary and $L$ is the maximum sequence length. 
The model outputs a BIO label sequence of the same length. Let $C$ denote the number of types. Subsequently, the number of BIO labels is $2C+1$.

{SA-NER} assumes a corpus as an unstructured knowledge, which is split into token sequences of length $L$, following the existing retrieval-augmented language model (LM) \cite{retro}, to store a large corpus efficiently. We retrieved $m$ pieces of knowledge, and concatenated them into $X$.
We fed the concatenated text $X^+ \in V^{(m+1)L}$ into the model.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S.1
\section{Related Work}
\yn{Here, we review
{NER}, which uses raw text as knowledge from four perspectives:
textual knowledge without structured knowledge, textual knowledge with in-domain structured knowledge, textual knowledge with general-domain structured knowledge, and textual knowledge by pre-training billion-scale LMs (\S \ref{ssec:rw_ner}).}
In addition, we review retrieval-augmented LMs (\S \ref{ssec:rw_rag}).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.1
\subsection{{NER} with Unstructured Knowledge}
\label{ssec:rw_ner}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Textual knowledge without structured knowledge.}
\yn{Researchers have utilized various clues to retrieve useful raw text for {NER}\@. Traditional {NER} models focus on the surrounding contexts \cite{ner_global1,ner_global2,ner_global3} and linked documents \cite{twitter_link} to capture non-local dependencies.}
Recently, neural {NER} models have benefited from
neighbor sentences 
\yn{to obtain better contextualized word representations \cite{ner_corpus2,ner_corpus1}. Moreover,
Other studies encoded knowledge contexts on entity types, such as questions, definitions, and examples taken from in-domain structured KBs (\textit{e.g.,} UMLS Meta-thesaurus) \cite{knowledgener,li-etal-2020-unified}.
In this study, we developed a generic method to retrieve useful raw text (unstructured text) for NER\@.}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Textual knowledge with in-domain structured knowledge.}
\yn{Distant supervision \cite{distant} uses} structured knowledge to annotate raw text with pseudo labels. Performing distantly supervised fine-tuning with in-domain structured knowledge after MLM pre-training is effective for domain-specific {NER} \cite{distant_chem,distant_bio}.
However, domain-specific distance-supervised learning \yn{depends on} 
structured knowledge coverage of the label set
of the downstream task. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Textual knowledge with general-domain structured knowledge.}
Weakly supervised learning with general domain-structured knowledge \cite{ner_weak,bond,gazetteer_nn2,ner_bert} transfers general domain knowledge to the target domain. These methods learn entity knowledge through weakly supervised learning, even though the target task has domain-specific entities and types \cite{ner_bert}.
We confirmed that our model achieved a performance gain by using raw text as unstructured knowledge at the inference time because world knowledge cannot be stored in a limited-size model.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Textual knowledge via pre-training of billion-scale LMs.}
\textcolor{black}{Pre-trained LMs memorize factual knowledge in their models through pre-training on unstructured corpus \cite{lm_kb1,lm_kb2,lm_kb3}. Recently, billion-scale generative pre-trained LMs have been proposed \cite{t5,gpt-3}. 
Although generative models cannot be naively applied to structured prediction tasks such as {NER}, some studies have addressed {NER} with generative LMs \cite{ner_gen1,ner_gen2,ner_gen3,ner_gen4}. An advantage of retrieval-augmented LMs over billion-scale LMs is the ease of maintenance; for instance, the models can use up-to-date Wikipedia as the UKBs.}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.2
\subsection{Retrieval-Augmented Language Models}
\label{ssec:rw_rag}

\paragraph{Retrieval for NLP tasks.}
LMs using external knowledge have recently been proposed \cite{realm,rag,fid,emdr,retro,atlas,nonpara_mlm}.
\textcolor{black}{The pioneering work, REALM (Guu et al. 2020), first encodes the original input and retrieves one piece of knowledge using the obtained sentence embedding as the query. The concatenated text of the original input and retrieved knowledge are then fed to the LM in REALM\@.
}
However, REALM and subsequent models focused on language modeling and ODQA, and successful retrieval-augmented LMs in {NER} have not been reported.
They obtained queries for knowledge retrieval such that each query represented the entire input or a fixed-length chunk split from the input.
Therefore, they cannot retrieve information that indicates 
the use of entities, which is important for {NER}\@.
In addition, as an input may include many entities, 
the model should focus only on the entities whose knowledge is not stored. However, retrieval-augmented LMs do not incorporate mechanisms to create and filter multiple queries.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Retrieval from training dataset.}
Some studies have found that retrieving knowledge from training data is also useful for sequence-to-sequence and extractive QA tasks, as it provides knowledge that is not stored in the trained model \cite{retreive_train1,retreive_train2}. Therefore, we implemented {SA-NER} such that it uses both labeled and unlabeled {UKB}s.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Retrieval from virtual knowledge base.}
TOME \cite{tome} uses a virtual knowledge base 
\yn{whose values are vector representations.
By focusing on entity knowledge, they extracted mentions from hyperlinks in Wikipedia to learn their representations. They reported that the virtual KB was less accurate but more
efficient than FID \cite{fid}, 
which reads input and textual knowledge with attention.}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4
\section{Method}
\label{sec:model}
Here, we presented {SA-NER}\@.
We explained the construction of the unstructured knowledge base (\S\ref{ssec:kb}), encoder architecture (\S\ref{ssec:encoder}), and the two-stage NER algorithm, which
revise the prediction using unstructured knowledge  (\S\ref{ssec:inf}), \textcolor{black}{fine-tuning} method (\S\ref{ssec:training}), and pre-training method (\S\ref{ssec:pre}).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.1
\subsection{Unstructured KB Construction}
\label{ssec:kb}
We created an unlabeled {UKB} from the raw text and a labeled {UKB} from the training data.
We assume in-domain \yn{text} 
as a source of unlabeled unstructured knowledge and split it into token sequences of length $L$, which is equal to the maximum length of the original inputs for SA-NER\@. 
Additionally, following a previous work \cite{retreive_train1}, we added the model's training data as labeled unstructured knowledge. We set $L=64$ to avoid truncating most original inputs.

Unstructured knowledge is stored in the {UKB}s with multiple associated keys.
The keys of the sequence are sentence and $n$-gram embeddings.
Here, we refer to previous work \cite{bert_sentemb} which indicated that the average of the token embeddings is more useful for sentence embedding than the first \texttt{[CLS]} embedding. It also demonstrated that the embeddings in the lower layers are also important, as well as those in the last layer.
Therefore, we define sentence embedding and $n$-gram embedding as the average pooling of token representations. 
The token representations are concatenations of the frozen BERT input and output such that both context-free and contextualized meanings are considered. 

To \yn{select only} entity-like $n$-grams as keys 
we remove $n$-grams that have stop words or no capital letters.
\textcolor{black}{We set $n=3$ in this paper. Note that only 0.5\% of the entities in the training data of CoNLL03, a widely used large-scale NER dataset, were filtered out using this method.}
String matching was used for filtering. 
We hold only \textcolor{black}{text chunks} including the $n$-grams present in the training data for the {UKB}s used during training time \textcolor{black}{for computational efficiency.} In addition, we hold \textcolor{black}{text chunks} including the $n$-grams present in the training or development (test) data for the {UKB}s at the inference stage on the development (test) data. 

Following previous work \cite{retreive_train1}, we use the labeled {UKB}, even in 
training to reduce the training-test discrepancy, the model does not retrieve the 
input itself from the labeled {UKB}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.2
\subsection{Encoder}
\label{ssec:encoder}
We used BERT \cite{bert} and a linear classifier with softmax activation as the encoder $f$.
Figure \ref{fig:model} illustrates the structure of the encoder.
To represent label information from the labeled knowledge base in the model, we provided additional token-type embeddings.
Although the token type is always zero in the conventional BERT model for {NER}, we used $2C+3$ token-type IDs.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{align}
t_i =
\begin{cases}
0 & \text{if $x_i$ is the original input} \\
1 & \text{if $x_i^+$ is unlabeled knowledge} \\
l_i + 2 & \text{if $x_i^+$ is labeled knowledge} \\
\end{cases},    
\end{align}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
where $l_i$ is the label of the labeled knowledge and $X^+$ is the concatenated text.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F2
\begin{figure*}
\begin{center}
\includegraphics{31-2ia4f2.pdf}
\end{center}
\hangcaption{\yn{Overview of our self-adaptive {NER} with 
knowledge retrieval from {UKB}s, which store text with $n$-gram and sentence embeddings as keys.
The labeled {UKB} has 
text with labels encoded as token type embeddings.} The queries are embeddings of unconfident entities and input. 
We use a sparse matrix in the self-attention modules in BERT.}
\label{fig:model}
\end{figure*}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


In the self-attention module, we used the sparse attention technique to reduce the space and time complexity from $\mathcal{O}(m^2L^2)$ to $\mathcal{O}(mL^2)$.
As shown in Figure \ref{fig:model}, we masked inter-knowledge interaction\footnote{We implemented $X^+$ as an $(m+1) \times L$ tensor. We calculate three attention matrices: intra-sequence attention ($(m+1) \times L \times L$), knowledge-to-input attention ($m \times L \times L$), and input-to-knowledge attention ($m \times L \times L$). This operation takes advantage of the parallel computing on the GPU\@.
}. Let $k$ be a function that returns zero as the sentence ID if the sequence is the input $X$ and $1,...,m$ if the sequence is knowledge.
Accordingly, the attention matrix before the softmax operation is: 
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{align}
A_{ij} =
\begin{cases}
\frac{\bm{Q}_i^\top \bm{K}_j}{\sqrt{d_k}} & \text{if $k(i) =k(j)$ or $k(i)k(j) =0$} \\
- \infty & \text{otherwise}
\end{cases},
\end{align}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
where $i,j$ are the token indices; $d_k$ is the number of dimensions of the attention head; and $\bm{Q}$ and $\bm{K} \in \mathbb{R}^{(m+1)L \times d_k}$ are the query and key matrices, respectively.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%Al1
\begin{algorithm}[t]
\caption{\yn{~Two-stage self-adaptive {NER}}}\label{alg} 
\input{04algo01.tex}
\end{algorithm}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.3
\subsection{Two-stage Tagging of Self-Adaptive {NER}}
\label{ssec:inf}
{SA-NER} performs two-stage tagging, \textit{i.e.,} calculation of $\bm{P}=f(X)$ and $\bm{P}^+=f(X^+)$.
The purpose of the first stage was to 
find entities that require additional information and 
obtain queries for knowledge retrieval.
The second stage is to \yn{refine} 
labels using retrieved knowledge.
The motivation behind this design was to retrieve useful entity-wise knowledge to disambiguate individual tokens in {NER}\@.
We predicted entity spans for entity-level retrieval.
We use only unconfident entities as \yn{entity-based} queries to exclude
\yn{unnecessary knowledge from the retrieved results.}
The pseudocode of the model is presented in Algorithm \ref{alg}.

We obtained the classification probabilities of the given text $\bm{P} =f(X) \in \mathbb{R}^{L \times (2C+1)}$ and the text with knowledge $\bm{P}^+ =f(X^+) \in \mathbb{R}^{L \times (2C+1)}$, where the vectors after position $L$ are ignored.
The model parameters are shared in two stages.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{First stage.}
We fed $X$ into the model to obtain the classification probability $\bm{P} \in \mathbb{R}^{L \times (2C+1)}$ \textcolor{black}{and collect unconfident entities $\mathcal{U}$ in $X$}.
We then extracted entities $\mathcal{E}$ from $X$ in accordance with the predicted labels $\hat{\bm{y}} =\mathrm{argmax}_c \bm{P}_{\cdot c} \in  \mathbb{N}^{L}$. 
The confidence score of a predicted entity $e$ is $c_e =\min_{i\in I_e} P_{i, \hat{y}_i}$, where $I_e$ is 
\yn{the span of 
$e \in \mathcal{E}$.}
If the type predictions are inconsistent in an entity (\textit{e.g.}, [B-LOC, I-PER]), we set $c_e =0$.
We collected unconfident entities $\mathcal{U} \subseteq \mathcal{E}$ whose confidence scores
are less than a 
threshold, $\lambda_{\mathrm{conf}}$.

Subsequently, we obtain the queries, which are sentence 
and entity embeddings.
Sentence embedding is the average pooling over all the token embeddings. 
Each unconfident entity $u \in \mathcal{U}$ contains multiple entity embeddings: the average-pooled vectors of $n$-grams that share at least one token with $u$.
$n$-grams were filtered out, similar to the {UKB} construction (\S \ref{ssec:kb}). 
$E$ denotes the number of 
\textit{entity embeddings} (which are embeddings of $n$-grams overlapping with $u \in \mathcal{U}$).
Each token embedding is a concatenation of the BERT inputs and outputs.
We considered only sentence-to-sentence and entity-to-$n$-gram matching.
We retrieved \textcolor{black}{the $m$ pieces of knowledge $K_1^m$, which are} values associated with the top-$m$ nearest neighbor sentence embeddings of the sentence embedding from the labeled and unlabeled UKBs.
\textcolor{black}{Similarly, we retrieved a total {\linebreak}of $2Em$ pieces of knowledge using the $E$ entity ($n$-gram) embeddings from the two sets of UKBs.}
We then selected the top-$m$ nearest knowledge from the collected $2(E+1)m$ knowledge while 
\yn{deduplicating} the backbone knowledge sequence by keeping the knowledge having the minimum distance.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Second stage.}
We concatenate knowledge $K_1^m$ with input $X$ and obtain the classification probability $\bm{P}^+ =f(X^+ = [X; K_1^m])$.
Finally, the model outputs BIO labels. 
\textcolor{black}{by operating
$\mathrm{argmax}~\bm{P}$ 
for the tokens in confident entities and 
$\mathrm{argmax}~\bm{P}^+$ for other tokens without any sequence-level decoding algorithms such as the Viterbi algorithm}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.4
\subsection{Training}
\label{ssec:training}
To train our two-stage {SA-NER}, we supervised the training data to refine unconfident entities and design a loss function.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Unconfident entity collection.}
In the training phase, we added misclassified entities, that is, those for which the prediction is incorrect, to the unconfident entities $\mathcal{U}$ described in \S\ref{ssec:inf}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Loss function.}
\yn{We used two cross-entropy losses, $\mathcal{L}_{1}$ for the model prediction without knowledge (the first step) and $\mathcal{L}_{2}$ for the model prediction with knowledge  (the second step).
The total loss function is $\mathcal{L}_{2} + \lambda_{1} \mathcal{L}_{1}$, where $\lambda_{1}$ is the hyperparameter.}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.5
\subsection{Pre-training}
\label{ssec:pre}
As is performed in retrieval-augmented language models for ODQA \cite{realm,retro}, we added a retrieval-augmented pre-training stage before the fine-tuning. We proposed two methods for NER-aware retrieval-augmented pre-training \textcolor{black}{for the SA-NER model.}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{CoNLL03 pre-training.} The first method uses a general domain NER dataset called CoNLL03 \cite{conll03}.
The model was pre-trained using the method described above 
(from \S\ref{ssec:kb} to \S\ref{ssec:training}).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{NERBERT pre-training.}
The second method involves large-scale self-supervised pre-training following {NERBERT} \cite{ner_bert}.
Although {UKB} in {SA-NER} and the pre-training data overlap in some cases, {SA-NER} can use the knowledge effectively by referring to it at the inference time.

The pre-training corpus used was Wikipedia\@. If consecutive words in the corpus have a hyperlink, they are labeled as entities. We categorized these entities using the DBpedia Ontology \cite{dbpedia}. If an entity exists in an ontology, it is categorized according to its type. If it does not exist or it belongs to multiple types, we categorize it to the special ``ENTITY'' type.

We split the corpus into fixed-length token sequences\footnote{Although the original NERBERT uses a sentence as a unit, we use a fixed length to share the setting with {UKB}s.} and 
\yn{extract the sequences with tokens labeled with the DBpedia types.}
We reduce the proportion of ``ENTITY'' labels by using filtering rules and down sampling.
The resulting dataset contains 33M examples, 939M tokens, and 404 types.

We added a final linear layer with a trainable parameter $\bm{W}_{\mathrm{pre}} \in \mathbb{R}^{d \times (2C_{\mathrm{pre}} +1)}$ to the top of the BERT, where $d$ is the hidden size of the BERT and $C_{\mathrm{pre}}$ is the number of types. Before fine-tuning, the final 
layer is replaced with a randomly initialized linear layer whose output dimensions are determined by the downstream task.
\yn{Refer to Appendix \ref{append:data} and the original paper \cite{ner_bert} for details.}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Knowledge retrieval.}
We used the {SA-NER} model in pre-training to \yn{reduce the pre-training and fine-tuning discrepancies.}
We used the pre-training data as {UKB}s, retrieved knowledge with pseudo-labels from the data as labeled knowledge, and 
randomly deleted the pseudo-labels to ensure that the knowledge is unlabeled. We set the deletion probability to $0.95$ to simulate downstream tasks in which the unlabeled {UKB} was larger than the labeled {UKB}.
For efficiency, 
we use Wikipedia hyperlinks as keys and queries for retrieval. Instead of a two-stage prediction, we sampled $m$ pieces of knowledge that included an entity in the original input.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5
\section{Evaluation}
We conducted experiments using three {NER} datasets to evaluate the effectiveness of the proposed self-adaptive {NER} with unstructured knowledge. Following the literature, we used entity-level F$_1$ as the metric. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.1
\subsection{Datasets}
\textbf{CrossNER} \cite{crossner} comprises five domains: politics, science, music, literature, and AI\@. This small-scale dataset was created by annotating sentences extracted from Wikipedia articles in each domain. It provides a textual corpus extracted from Wikipedia for in-domain pre-training. This was used for the unstructured \textcolor{black}{{UKB}\@.} {The} label sets are different among the domains. 
The data statistics are presented in Table \ref{tab:data}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T1%
\begin{table}[t]
\input{04table01.tex}
\hangcaption{Data statistics. UKB indicates the size of UKB (MB).}
\label{tab:data}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\textbf{Finance} \cite{finance_ner} 
is a medium-scale {NER} dataset 
obtained from U.S\@. 
SEC filings. We used Wikipedia articles in the finance domain as the textual corpus $\mathcal{D}$ to construct the unlabeled UKB\@. The label set was \yn{person, organization, location, and miscellaneous}.

\textbf{CoNLL03} \cite{conll03} is a widely used large-scale {NER} dataset collected from Reuters news stories between August 1996 and August 1997. 
We used the Reuters-21578 text classification dataset \cite{reuter21578} collected from Reuters in 1987 as $\mathcal{D}$.
The label set was the same as that used for financing.
We used this general-domain large-scale dataset only for pre-training.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.2
\subsection{Compared Models}
Our text encoder and tokenizer are pre-trained BERT-base-cased  \cite{bert} or DistilBERT-base-cased models  \cite{distilbert}. 
\textcolor{black}{We used the pre-trained models without fine-tuning to obtain both sentence and entity embeddings. We shared and fine-tuned the parameters of the NER models used in the first and second steps.}
All experiments used the hyperparameters determined from the development set of CrossNER-Politics; refer to Appendix \ref{append:hyper}. 

We pre-trained the compared models on {CoNLL03} or {NERBERT} \cite{ner_bert}\footnote{\yn{Our implementation was different from the original {NERBERT} regarding the fixed-length sequences, initialization, loss function, and data collection results; refer to Appendix \ref{append:data}.}} Datasets before fine-tuning. In addition to the BERT model (\textit{i.e.}, BERT with {CoNLL03} or {NERBERT} pre-training), we implemented the {NER} version of {REALM} ({REALM}-{NER})\@.
For {REALM}{NER}, we replaced the retrieval-augmented MLM of {REALM} with 
our retrieval-augmented pre-training methods tailored for {NER} to assess the effectiveness of our knowledge retrieval. In addition, we set $m =1$, removed entity-level retrieval, and ignored the labeled {UKB} \textcolor{black}{because REALM retrieved one piece of knowledge with sentence embedding from the unlabeled UKB}.
We cited the results of the previous models: BERT, {NERBERT}, and DAPT \cite{tapt}, which is the domain-adapted BERT baseline.\footnote{We did not cite the results of {NERBERT} on Finance because the authors did not report the data splits.} We compared our model with models comprising BERT and a linear classifier because the classifier architecture was beyond the scope of our study.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T2
\begin{table*}[b]
\input{04table02.tex}
\hangcaption{Main results on the test set. \# Train means the number of the training instances and \# Types means the number of the named entity types. Therefore, it is more difficult to learn domain knowledge from lefter datasets. CN indicates CoNLL03, and NB indicates \textcolor{black}{NERBERT}. The models were pre-trained from BERT-base-cased. We ran five experiments with different seeds. Standard deviations are parenthesized. Performances of the previous models are cited from the original CrossNER paper.$^\dag$}
\label{tab:main_result}
\end{table*}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T3
\begin{table*}[b]
\input{04table03.tex}
\hangcaption{Main results on the test set. The models were pre-trained from DistilBERT-base-cased.}
\label{tab:main_result_distil}
\end{table*}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


We provide the results of the models pre-trained on CoNLL03 and NERBERT\@. The model pre-trained on CoNLL03 learned only NER capability through pre-training, whereas the others learned the knowledge of many entities.
However, the NERBERT and CrossNER corpora overlap between entities and their supervision because both corpora depend on Wikipedia. Through experiments, we confirmed that overlapping entities did not affect the effectiveness of SA-NER through experiments; refer to Table \ref{tab:overlap}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.3
\subsection{Main Results}

Tables \ref{tab:main_result} and \ref{tab:main_result_distil} present the primary results.
The proposed model outperformed the baselines across all target domains, models, and pre-training datasets.
The improvement is typically larger in lower-resource domains with more types because \yn{per-type supervision is limited in such cases.}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{\yn{Does self-adaptive {NER} improve the performance of the NER-aware pre-training?}}
{SA-NER} outperformed BERT with {CoNLL03} and \textcolor{black}{NERBERT} pre-training. This indicates that self-adaptation using unstructured knowledge at the inference time obtains domain knowledge that is not stored in the model during pre-training in the general domain.
Moreover, because we can increase the unlabeled {UKB} without annotation, the model can acquire new knowledge more efficiently through self-adaptation than through training.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Does self-adaptive {NER} improve the performance of the retrieval-augmented LM baseline?}
{SA-NER} outperformed {REALM}-{NER}.
{SA-NER} retrieves knowledge with entity-level retrieval from labeled and unlabeled {UKB} and encodes large pieces of knowledge owing to sparse attention. 
These techniques improve the usefulness of the knowledge for {NER}\@. The contributions of each component are discussed in ablation studies.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.4
\subsection{Ablation Studies}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T4
\begin{table}[b]
\input{04table04.tex}
\hangcaption{Ablation studies on the development set of the politics domain. $\Delta$ shows the drop from the proposed model. 
Each ablation was conducted in the fine-tuning and evaluation.}
\label{tab:ablation}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T5
\begin{table}[t]
\input{04table05.tex}
\hangcaption{The distribution of the number of knowledge sentences retrieved by the sentence embedding among $m=10$ knowledge sentences. `10 (with unconfident entities)' indicates that all knowledge sentences were retrieved with the sentence embedding although an unconfident entity existed in the original input, and vice versa.}
\label{tab:pie}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Table \ref{tab:ablation} presents the results of ablation studies. 
In the subsequent sections, we focus on the political domain, for computational reasons. We note that the politics domain is \textcolor{black}{the} hardest setting for SA-NER because it has the largest training data and the fewest entity types.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Does the entity-level retrieval improve performance?}
First, we confirmed the usefulness of self-adaptive knowledge retrieval because knowledge retrieval based on the model's entity prediction is more useful for {NER} than for conventional sentence-level retrieval ($\Delta 1.01$ vs.\ $\Delta 0.31$ for CoNLL03 and $\Delta 1.12$ vs.\ $\Delta 0.79$ for NERBERT). 
We found that sentence-level retrieval alone did not improve the performance of the BERT baseline. However, sentence-level retrieval affects performance. 
\textcolor{black}{Table \ref{tab:pie} shows the number of the knowledge sentences retrieved by sentence embeddings among ten sentences. We found that entity-level retrieval is valid for only 39.0\% of instances and unconfident entities do not exist for 55.6\% of instances.
We consider entity-level retrieval to be useful but not dominant, and sentence-level retrieval to be dominant but rarely useful. Thus, using both retrievals was the most effective.
} 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Does the distinction about confidence improve the performance?}
Second, we investigated the efficacy of distinguishing the predicted entities in terms of confidence.
The model retrieves knowledge regarding unconfident entities $\mathcal{U} =\{e | c_e < \lambda_{\mathrm{conf}}, e \in \mathcal{E} \}$, 
and then refines the prediction for only the unconfident entities using the retrieved knowledge.
We set $\lambda_{\mathrm{conf}} >1$ to eliminate the distinction.
We observed that ignoring confident entities when creating queries was slightly effective ($\Delta 0.42$) for the model pre-trained on NERBERT and not \textcolor{black}{effective} ($\Delta 0.16$) for the model pre-trained on CoNLL03. We can restrict the retrieval results to informative knowledge for {NER} using this distinction, which is useful for a pre-trained model with rich entity information. However, a pre-trained model with poor entity information does not require such a distinction.
We then used the second-step prediction for all tokens. We found that by reusing the first-step prediction for confident entities, 
performance is improved ($\Delta$ 0.53 for CoNLL03 and $\Delta 0.36$ for NERBERT). This is because the UKB included noise, and noisy knowledge harmed the prediction in some cases.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Do the labeled and unlabeled {UKB}s improve the performance?}
Finally, we confirmed that unlabeled {UKB} was important ($\Delta 0.93$ for CoNLL03 and $\Delta 1.10$ for NERBERT) for SA-NER\@. However, the labeled UKB was useful only for the model trained on NERBERT ($\Delta$ 0.51).
We consider that in the low-resource NER task, knowledge in the training data can be stored in the model during fine-tuning if the pre-trained model does not have rich entity information. However, retrieval from the training data is useful for models with rich entity information 
\textcolor{black}{because not all knowledge in the training data can be stored in the model.}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F3
\begin{figure}[b]
\begin{center}
\includegraphics{31-2ia4f3.pdf}
\end{center}
\hangcaption{F$_1$ score versus the amount of knowledge $m$ of the model pre-trained on CoNLL03 and \textcolor{black}{fine-tuned} in the politics domain. The error bars show the standard deviation over five runs. For a fair comparison of the time and space complexity, we compared the methods with different values of $m$, since our sparse attention runs in $\mathcal{O}(ml^2)$ while the dense attention runs in $\mathcal{O}(m^2l^2)$.}
\label{fig:num_conll}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F4
\begin{figure}[b]
\begin{center}
\includegraphics{31-2ia4f4.pdf}
\end{center}
\hangcaption{F$_1$ score versus the amount of knowledge $m$ of the model pre-trained on NERBERT and \textcolor{black}{fine-tuned} in the politics domain.}
\label{fig:num_nerbert}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.5
\subsection{Discussion}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Does the performance of our model depend on the amount of knowledge?}
Figures \ref{fig:num_conll} and \ref{fig:num_nerbert} plot F$_1$ score against the amount of knowledge $m$.
We observed that more knowledge leads to higher F$_1$ scores. Because the time and space complexities of sparse attention are linear in terms of the number of pieces of knowledge, sparse attention is suitable for large $m$. However, dense attention did not improve performance in the case of large $m$. We consider sparse attention to represent intra- and inter-sequence interactions more effectively than naive dense attention.
We also found that the results of the model \textcolor{black}{pre-trained} on CoNLL03 did not leverage the 16 knowledge sentences. We consider it important to pretrain with many long-tail entities to acquire the capability to use many knowledge sentences.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T6
\begin{table}[b]
\input{04table06.tex}
\hangcaption{Results for each entity type on the development set in the politics domain.}
\label{tab:discussion}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{\textcolor{black}{Does the supervision of the entity affect the effectiveness of SA-NER?}}
Table \ref{tab:discussion} lists the results when the target entities are restricted to each type, which are defined in terms of whether the supervision of an entity is included in the \textcolor{black}{fine-tuning} and pre-training data.
The proposed model outperformed BERT for all types.
The improvement was 0.65 points on CoNLL03 and 1.15 points on NERBERT for the ``seen in \textcolor{black}{fine-tuning}'' type, and 1.17 points on CoNLL03 and 1.64 points on NERBERT for the ``unseen in \textcolor{black}{fine-tuning}'' type.
Therefore, self-adaptation has an effect regardless of whether the entity exists in the training data. We also observed this effect in ablation studies.

Regarding the ``unseen in pre-training'' type, the proposed model improved the performance by 3.28 points on NERBERT. We found that \textcolor{black}{\textit{pre}}-training data are \textcolor{black}{more} valuable than previously thought, similar to the findings of \cite{retreive_train1} that the reference to the training data at inference time is worthwhile. By contrast, CoNLL03 had fewer domain-specific entities. Thus, the entities unseen in pre-training can be classified more accurately by SA-NER\@.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Is the self-adaptive {NER} sensitive to the unconfidence threshold?}
To investigate the sensitivity of SA-NER to the hyperparameter, we set $\lambda_{\mathrm{conf}}$ to various values at the inference time after the training with $\lambda_{\mathrm{conf}} =0.9$. We investigated the model pre-trained on NERBERT because the improvement using the threshold was small in the CoNLL03 setting.

Table \ref{tab:robustness} presents the results. The performance is on par if  $\lambda_{\mathrm{conf}} \in [0.8, 0.95]$. Therefore, the SA-NER is not sensitive to $\lambda_{\mathrm{conf}}$. We also confirmed that modifying the prediction of high-confidence entities is harmful ($\lambda_{\mathrm{conf}} =1$) and thus using $\lambda_{\mathrm{conf}}$ is useful. Moreover, we observed that modifying the predictions of certain entities (3.6\% of the total number) is important (\textcolor{black}{$\lambda_{\mathrm{conf}} =0.1$}). These entities are those in which the token-level predictions are inconsistent, and their confidence $c_e$ is set to zero. \textcolor{black}{We changed the threshold at the inference phase alone owing to the large training costs. Because the training of the model with the same threshold reduces the train-test discrepancy, a setup with a large difference between $\lambda_{\mathrm{conf}}$ in the training and inference degrades the performance. Thus, we conclude that the SA-NER is robust, at least in $\lambda_{\mathrm{conf}} \in [0.8, 0.95]$.}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T7
\begin{table}[t]
\input{04table07.tex}
\hangcaption{\textcolor{black}{F1 score versus confidence threshold $\lambda_{\mathrm{coef}}$ of the model pre-traind on NERBERT and \textcolor{black}{fine-tuned} in the politics domain. Unconfident proportion indicates the proportion of unconfident entities to all entities. We omitted rows $0.2, 0.3, 0.4,$ and $0.6$, whose performance is the same as that of the rows directly above.}}
\label{tab:robustness}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{What determines the confidence score?}
In this study, we investigated the factors that determine confidence scores. We used a model pre-trained on NERBERT for the same reason.
We calculated the Spearman rank correlation coefficient for each factor using the confidence score. As shown in Table \ref{tab:robustness}, the confidence scores of 68.01\%($=100\%-31.99\%$) of the entities are distributed over 0.99. 
To analyze the confidence scores of entities that are practical targets of retrieval augmentation, we removed such dominant but non-target entities from the calculation.

Table \ref{tab:correlation} presents the results. 
\textcolor{black}{
We found that the entity frequency in the pre-training and fine-tuning corpora was uncorrelated or only slightly correlated with the confidence score.}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T8
\begin{table}[t]
\input{04table08.tex}
\hangcaption{Spearman rank correlation coefficient to the confidence score of the model pre-traind on NERBERT and \textcolor{black}{fine-tuned} in the politics domain.}
\label{tab:correlation}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T9
\begin{table}[t]
\input{04table09.tex}
\hangcaption{Performance on the development set. The models were pre-trained on CoNLL03, NERBERT without the entity overlap, and NERBERT with the entity overlap.}
\label{tab:overlap}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Moreover, we observed that whether the entity prediction is correct is \textcolor{black}{weakly} correlated with its confidence score \textcolor{black}{with high significance}. 
Therefore, the confidence score \textcolor{black}{ indicates } the correctness of the prediction accuracy. This observation justifies the use of confidence scores in SA-NER.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{What is the effect of overlapping entities between CrossNER and NERBERT?}
To investigate the effect of overlapping entities in the NERBERT pre-training and fine-tuning datasets, we conducted experiments in which we removed sequences including entities that appeared in the CrossNER dataset from the NERBERT corpus. Table \ref{tab:overlap} presents the results. We confirmed that the NER ability learned from the NERBERT corpus itself improved the performance and that SA-NER outperformed NERBERT in both settings.

Moreover, the improvement in the SA-NER did not depend on the setting. Thus, we concluded that the effectiveness of SA-NER was not affected by overlapping entities.

We also found that the performance of NERBERT was overestimated because of the entity overlap. A previous study \cite{gpt-3,case_study_c4} also noted that the leakage of benchmark datasets from the pre-training corpus affects the performance of GPT-3 \cite{gpt-3} and T5 \cite{t5}. In future, the community should address this problem.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.6
\subsection{Qualitative Analysis}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T10
\begin{table}[t]
\input{04table10.tex}
\hangcaption{Qualitative Analysis. 
\yn{One representative piece of knowledge retrieved for the input is provided.}}\label{tab:qualitative}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Table \ref{tab:qualitative} presents examples from the proposed model. The first example is a case in which the self-adaptation improved the model prediction.
The original inputs do not provide evidence that `the House of Freedoms' is a political party. However, knowledge provides evidence by mentioning it in the context of elections.
The second is the most common fault in politics. Owing to the imbalance between the training labels of persons and politicians, person entities tend to be misclassified as political entities. Although both the input and knowledge indicate that `Bob Weinstein' is not a politician, the model made the wrong prediction.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6
\section{Conclusion}
We proposed {SA-NER}, which was designed for {NER} to retrieve knowledge from labeled and unlabeled {UKB}s using unconfident entities and given inputs as queries. It efficiently encodes many pieces of knowledge with sparse attention.
In experiments, {SA-NER} outperformed DistilBERT and BERT baselines pre-trained on the {CoNLL03} and NERBERT datasets by 1.22 to 2.35 points. We found that entity-level retrieval, unlabeled {UKB}, and large $m$ enabled by sparse attention all contribute to {SA-NER}'s performance. In addition, the focus on unconfident entities and labeled UKB is crucial if the model has rich entity knowledge.

We believe that {SA-NER} can help application providers develop {NER} services in their target domain with the domain-specific entity types they have defined, even if they do not have a sufficiently annotated dataset.


% Start bibliography
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.7}
\bibliography{04refs}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\appendix
\section{Implementation}
\label{append:hyper}
The hyperparameter settings are listed in Table \ref{tab:hyper1}. 
Pre-training required 17~h on eight NVIDIA Quadro RTX 8000 (48GB) GPUs.
The training of the largest CoNLL03 dataset required 6~h on a GPU\@.
We set the number of early stop epochs to five in CoNLL03 for computational efficiency.
We used the Adam optimizer \cite{adam}, PyTorch (ver.~1.10.1)\footnote{\url{https://pytorch.org/}} \cite{pytorch} and transformers (ver.~4.15.0)\footnote{\url{https://github.com/huggingface/transformers}} \cite{transformers}.
Stop words were implemented using NLTK (ver.~3.7)\footnote{\url{https://www.nltk.org/}} \cite{nltk}.
We used faiss software (ver.~1.7.2)\footnote{\url{https://github.com/facebookresearch/faiss}} \cite{faiss} for the nearest-neighbor search for knowledge retrieval.
We set $L =64$ for all data preprocessing with a sliding window size of 16.
For the entities in the sliding window, we use the maximum operation to select from the two predictions.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T11
\begin{table}[t]
\input{04table11.tex}
\hangcaption{Hyperparameters.}
\label{tab:hyper1}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Our implementation of {NERBERT}}
\label{append:data}

\paragraph{Data Collection}
We used the Wikipedia dump on January 27, 2022 and the DBPedia Ontlogy dump on \ December 1\  2021.\footnote{We used en-specific data, which means that the types are annotated without transitive augmentation. \url{https://databus.dbpedia.org/dbpedia/mappings/instance-types/}}
We then split the corpus into fixed-length token sequences and removed the sequences without entities that were not labeled as ``ENTITY.'' 

We reduced the proportion of ``ENTITY'' labels by using filtering rules and down sampling.
We randomly filtered the sentences to reduce labels. If all entities in a sentence were the top-20 frequent labels, the sentences were randomly removed from the dataset: 30\% if the number of ``ENTITY'' entities was three, 50\% if the number was four, and 70\% if the number was more than four.
Weighted sampling was used during the pre-training period.
The sampling weight of the sentence is $\min_{0 \leq i \leq l} |E_{c_i} |^{-0.3}$, where $E_c$ is the number of entities of type $c$ in the dataset and $c_i$ is the type of the $i$-th token. 
The final dataset contained 33M examples, 939M tokens, and 404 types.\footnote{NERBERT reported their data has 16.3M examples, 457.6M tokens, and 315 types. However, they had not published their data or the URLs of the dump data before our experiments.}
Except for the loss function, initialization, and use of the retrieval-augmented model, we followed the procedure of the {NERBERT} pre-training algorithm.

\paragraph{Loss Function}
In addition to the cross-entropy loss used in the original {NERBERT}, we incorporated a multi-task loss to efficiently learn the {NER} ability by ignoring the very frequent ``ENTITY'' type in the entity typing.
For entity extraction, we performed three-class classification.
We summed the output probabilities of the final linear layer after the softmax activation to obtain the probabilities of ``B-[type],'' ``I-[type],'' and ``O.'' 
In the entity typing, we masked the output logits of the final linear layer corresponding to the ``ENTITY'' label. We then performed a $2C_{\mathrm{pre}} -1$-classification task.
The total loss is the sum of two cross-entropy losses.

\paragraph{Initialization}
We had to initialize the weight of the final linear layer and token-type embeddings because of the mismatch in the set of labels between the downstream and pre-training tasks.
Instead of a random initialization from $\mathcal{N}(0, \sigma_0)$, where $\sigma_0 \in \mathbb{R}$ is a fixed standard deviation, we used the learned distribution $\mathcal{N}(\bm{\mu}, \bm{\sigma})$, where $\bm{\mu}, \bm{\sigma} \in \mathbb{R}^d$ is the bias and the standard deviation of the weight of the final linear layer and the token-type embeddings in the pre-trained model.

% Start biography
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Kosuke Nishida}{
He has worked for NTT Human Informatics Laboratories since 2017 and is currently Associate Distinguished Researcher. He received his Ph.D. from the University of Tokyo in 2024. His research interests include natural language processing, and especially mathematical understandings of models and training algorithms for it.
}
\bioauthor[:]{Naoki Yoshinaga}{%
He received his Ph.D. from the University of Tokyo in 2005. He has been a JSPS Research Fellow (DC1, PD) from 2002 to 2008 and Associate Professor at the University of Tokyo since 2016. His research interests include computational linguistics and machine learning, and particularly efficient and adaptive natural language processing in the wild.
}
\bioauthor[:]{Kyosuke Nishida}{%
He received his Ph.D. degrees in information science and technology from Hokkaido University in 2008. He joined NTT Corporation in 2009 and is presently a Senior Distinguished Researcher at NTT Human Informatics Laboratories. His current interests include natural language processing, vision-and-language, and artificial intelligence.
}

\end{biography}

\biodate   %%%% Following settings will be edited by editors.


\end{document}


