\documentclass[english]{jnlp_3.8}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{array}
\allowdisplaybreaks

\usepackage{hangcaption_jnlp}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition

\usepackage{enumitem}
\usepackage{multirow, multicol}
%%%%%\usepackage{fancyvrb}
%%%%%\usepackage{forest}
%%%%%\usepackage{booktabs}
%%%%%\usepackage{caption}
%%%%%\usepackage{subcaption}
%%%%%\usepackage{lipsum}
%%%%%\usepackage{setspace}
%%%%%\usepackage{arydshln}

\makeatletter
\let\MYcaption\@makecaption
\makeatother

\usepackage{subcaption}
\captionsetup{compatibility=false}

\makeatletter
\let\@makecaption\MYcaption
\makeatother
\setlength{\captionwidth}{\textwidth} %%%% subcaption かつ hangcaption の時

\DeclareTextFontCommand{\textbfit}{\bfseriesitshape}

\usepackage{siunitx}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{32}
\Number{1}
\Month{March}
\Year{2025}
%%%%\Issuetitle{「　　　　」特集号}

\received{2024}{8}{1}
\revised{2024}{11}{11}
\accepted{2024}{12}{23}



\setcounter{page}{300}



%%%%   Title, Author, etc.            %%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Negation Scope Conversion for \\Unifying Negation-Annotated Datasets}
\eauthor{Asahi Yoshida\affiref{affi_1} \and Yoshihide Kato\affiref{affi_2} \and Shigeki Matsubara\affiref{affi_1}\affiref{affi_3}} 
\eabstract{Negation scope resolution is a technique that identifies the part of a sentence affected by the negation cue. The three major corpora used for it, the BioScope corpus, the SFU review corpus, and the Sherlock dataset, have different annotation schemes for negation scope. Due to the different annotations, it is difficult to use the three corpora together in the study of negation scope resolution. To address this issue by merging the corpora into a unified dataset based on a common annotation scheme, we propose a method for automatically converting the scopes of BioScope and SFU to those of Sherlock. We conducted an experiment to evaluate the accuracy of our method using a dataset obtained by manually annotating the negation scopes to a tiny portion of BioScope and SFU, verifying that our method can convert the scopes with high accuracy. In addition, we conducted another experiment to verify the effectiveness of our method from a pragmatic perspective, where we fine-tuned PLM-based negation scope resolution models using the unified dataset obtained by our method. The results demonstrated that the performances of the models increase when fine-tuned on the unified dataset, unlike the simply combined one, which supports the effectiveness of our method.}
\ekeywords{Negation, Negation Scope Conversion, Negation Scope Resolution}

\headauthor{Yoshida et al.}
\headtitle{Negation Scope Conversion for Unifying Negation-Annotated Datasets}



%%%%     Affiliation     %%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\affilabel{affi_1}{}{Graduate School of Informatics, Nagoya University}
\affilabel{affi_2}{}{Information \& Communications, Nagoya University}
\affilabel{affi_3}{}{Information Technology Center, Nagoya University}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

\renewcommand{\thefootnote}{\fnsymbol{footnote}}
\footnote[0]{This paper is an extended version of our preliminary paper \cite{yoshida-etal-2024-negation-scope}, which was presented at the 2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation (LREC-COLING 2024).}
\renewcommand{\thefootnote}{\arabic{footnote}}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S1
\section{Introduction}
Negation is a frequently occurring linguistic phenomenon in natural languages.
Accurate negation processing is crucial for natural language processing (NLP) systems because it reverses the meaning of sentences, phrases, words, etc.
Over the past decade, many researchers in the NLP community have addressed the work on negation processing.
Despite these extensive efforts, recent studies have shown that pre-trained language models (PLMs) still struggle with negation \cite{hossain-etal-2022-analysis,chen-etal-2023-say,truong-etal-2023-language,ye-etal-2023-assessing,garcia-ferrero-etal-2023-dataset,weller-etal-2024-nevir}.
The negation handling ability of language models has been evaluated from various perspectives, and some studies have pointed out that language models are not robust to variations in negation scope, which is the part of a sentence affected by the negation \cite{ravichander-etal-2022-condaqa,she-etal-2023-scone}.
Considering this point, this study addresses the fundamental technique of negation modeling, \textbf{negation scope resolution}: resolving the part(s) of a sentence affected by the negation cue (words that express the meaning of negation such as ``not'' and ``no'').

Many researchers have addressed negation scope resolution using various approaches; however, this remains a difficult problem due to its complexity.
When using negation-annotated corpora for the study of negation scope resolution, one issue arises: 
The three primary negation-annotated corpora, 
the BioScope corpus \cite{szarvas-etal-2008-bioscope}, the SFU review corpus \cite{konstantinova-etal-2012-review}, and the Sherlock dataset \cite{morante-daelemans-2012-conandoyle}, 
adopt different annotation schemes for the negation scope.
Table~\ref{tab:corpus-cmp} shows different annotations of negation cue and scope for the sentence ``The park allows fishing in designated areas only and does not allow swimming.''.
Due to this difference in scope annotation, it is difficult to use the three corpora together in the study of negation scope resolution.
For example, previous studies that performed negation scope resolution using PLMs trained a model on a single corpus instead of on a combined dataset consisting of the three corpora (e.g., \citeA{truong-etal-2022-improving}), because a simple combination of the three corpora can decrease the model performance due to the different annotations of the negation scope.
This issue can be resolved by merging the three corpora into a unified dataset based on a common annotation scheme.
One possible approach for creating a unified dataset is to manually re-annotate the three corpora according to a common annotation scheme.
However, such manual annotations require expert linguists and considerable time and effort.
Automated conversion of negation scopes can be an alternative solution.
However, to the best of our knowledge, no previous study has developed such a method.



%%%%%%%%%%%%%%%%%%%%%T1
\begin{table}[b]
{\vskip -5pt}
\input{11table01.tex}
\hangcaption{
Negation cue and its scope for the example sentence according to the three annotation schemes. The bold and underlined parts represent the negation cue and scope, respectively.}
\label{tab:corpus-cmp}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%



Inspired by these issues, this study proposes a method for automatically converting the scopes of BioScope and SFU to those of Sherlock.\footnote{Our code is available at \url{https://github.com/asahi-y/negation-scope-conversion}. The unified dataset can be obtained by applying our code to BioScope and SFU, and merging them with Sherlock. } We select the Sherlock scope annotation as the target for conversion because it can represent more complex negation scopes than BioScope and SFU. The key point of our method is to effectively convert scopes by utilizing the existing scope annotations in BioScope and SFU.

An experiment was conducted to evaluate the accuracy of the proposed method.
We obtained a dataset by manually annotating the negation scopes to a tiny portion of BioScope and SFU following the Sherlock standard.
Using this dataset, we experimentally evaluated the accuracy of our method and performed a qualitative error analysis.
The results demonstrated that our method can convert the scopes of BioScope and SFU to those of Sherlock with high accuracy.

Additionally, we conducted another experiment to verify the effectiveness of the proposed method from a pragmatic perspective.
We converted BioScope and SFU using our method, and merged Sherlock and the converted versions of BioScope and SFU into a unified dataset.
The unified dataset can be treated as a large training dataset for negation scope resolution models.
We fine-tuned the PLMs on the unified dataset and evaluated the model performances.
The experimental results demonstrated that the dataset created using our method improves the performance of the scope resolution model, unlike the simply combined one.

The remainder of this paper is organized as follows:
Section~\ref{sec:related-work} describes the related work of our study.
Section~\ref{sec:conversion} proposes a method for automatically converting the negation scopes of BioScope and SFU to those of Sherlock.
Section~\ref{sec:eval-conversion} reports an experiment demonstrating that our method can convert the scopes of BioScope and SFU to those of Sherlock with high accuracy.
Section~\ref{sec:data-aug} reports an experiment of negation scope resolution using the unified dataset obtained by the conversion method, demonstrating the better performance of the model trained on the unified dataset than the one on the simply combined dataset.
Finally, Section~\ref{sec:conclusion} concludes the paper.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2
\section{Related Work} \label{sec:related-work}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2.1
\subsection{Negation-Annotated Corpora} \label{subsec:neg-corpora}
This section describes the three negation-annotated corpora used in this study.

%%%%%
\paragraph*{BioScope}
The BioScope corpus \cite{szarvas-etal-2008-bioscope}\footnote{\url{https://rgai.inf.u-szeged.hu/node/105}} contains biological texts in which negation cues are annotated with their scopes. Sentences \ref{snt:bio} and \ref{snt:bio-passive} show examples annotated with negation cues (marked in bold) and their scopes (underlined).
%%%%%
{\vskip.5\baselineskip}
\begin{enumerate}[label=(\arabic*)]
  \item The transcription factors did \underline{\textbf{not} change}. \label{snt:bio}
  \item \underline{The feature was \textbf{not} seen in the resting cells}. \label{snt:bio-passive}
\end{enumerate}
{\vskip.5\baselineskip}
%%%%%
Basically, the scope annotation covers only the right side of the cue.
Exceptional cases exist in which the scope annotation also covers the left side of the cue (e.g., a passive sentence is such a case, as shown in Sentence~\ref{snt:bio-passive}).

%%%%%%
\paragraph*{SFU}
The SFU review corpus \cite{konstantinova-etal-2012-review}\footnote{\url{https://www.sfu.ca/~mtaboada/SFU_Review_Corpus.html}} is a collection of reviews that belong to 8 different domains such as books, cars, and hotels.
Most annotation schemes of negation scope follow those of BioScope.

%%%%%%
\paragraph*{Sherlock}
The Sherlock dataset \cite{morante-daelemans-2012-conandoyle}\footnote{
  \url{https://www.clips.ua.ac.be/sem2012-st-neg/data.html}
} contains Conan Doyle stories annotated with negation cues and their scopes.
Sentences \ref{snt:sherlock-1}, \ref{snt:sherlock-2}, and \ref{snt:sherlock-3} present annotation examples.
%%%%%
{\vskip.5\baselineskip}
\begin{enumerate}[resume, label=(\arabic*)]
  \item \underline{We did} \textbf{not} \underline{drive up to the door}.
  \label{snt:sherlock-1}
  \item You'll see \underline{how} \textbf{im}\underline{possible for me to go there}.
  \label{snt:sherlock-2}
  \item \underline{An investigator needs} facts and \textbf{not} \underline{legends or rumours}.
  \label{snt:sherlock-3}
\end{enumerate}
{\vskip.5\baselineskip}
%%%%%
The annotation guidelines of Sherlock \cite{morante-etal-2011-guideline} are based on those of BioScope; however, several improvements have been made.
The main improvements are as follows:
%%%%%
\begin{itemize}
  \item The scope includes all the arguments of the event being negated (e.g., the scope in Sentence \ref{snt:sherlock-1} includes the subject ``we'').
  \item Affixal negation cues are also considered (e.g., in Sentence \ref{snt:sherlock-2}, the affixal cue ``im'' and part of its scope ``possible'' are distinguished).
  \item Discontinuous scopes are allowed (e.g., an elliptical construction is such a case, as shown in Sentence \ref{snt:sherlock-3}).
\end{itemize}
%%%%%

These improvements have made it possible to represent more complex negation scopes. {\linebreak}\citeA{fancellu-etal-2017-detecting} pointed out that the scope annotation of Sherlock is linguistically motivated.
Consequently, this study explores a method for converting the scopes of BioScope and SFU to those of Sherlock.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2.2
\subsection{Negation Scope Resolution} \label{subsec:neg-scope-resolution}
This section explains the negation scope resolution methods used as the basis for our conversion method.
We selected these methods due to their high performance.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2.2.1
\subsubsection{Syntax-Based Negation Scope Resolution} \label{subsubsec:syntax-based-nsr}

One method that we adopt is the syntax-based negation scope resolution method proposed by \citeA{yoshida-etal-2023-revisiting}.
They modified the syntax-based method proposed by \citeA{read-etal-2012-uio1}.
Yoshida et al.'s method follows these steps:
%%%%%
\begin{enumerate}[label=\arabic*.]
  \item Parse the sentence and select the constituents that dominate the cue as candidates.
  \item From the candidates, select one constituent corresponding to the scopes using the heuristics.
  \item Adjust the scope by removing certain elements from the constituent.
\end{enumerate}
%%%%%
As an example, let us consider Sentence \ref{snt:he-did-not} with a negation cue.
%%%%%
{\vskip.5\baselineskip}
\begin{enumerate}[start=5, label=(\arabic*)]
  \item \underline{He did} \textbf{not} \underline{go to school} and stayed home. \label{snt:he-did-not}
{\vskip.5\baselineskip}
\end{enumerate}
%%%%%

In Step~1, the method obtains the parse tree, as shown in Figure~\ref{tree:he-did-not}, and selects the RB, two VPs, and S as the negation scope candidates.
In Step~2, the constituent S is selected using the path pattern rules.
Step~3 removes the coordinating conjunction ``and,'' its conjunct ``stayed home,'' and the punctuation ``.'' according to the adjustment rules (we explain the details of the new version of the path pattern rules and the adjustment rules in the next section).
Using a series of processes, the scope ``He did, go to school'' is correctly resolved.


%%%%%%%%%%%%%%%%%%%%%%%%%%F1
\begin{figure}[b]
{\vskip -17pt}
\begin{center}
\includegraphics{32-1ia11f1.eps}
\end{center}
\hangcaption{Parse tree of Sentence~\ref{snt:he-did-not}. Underlined constituents show scope candidates. Enclosed parts are removed in the adjustment step.}
\label{tree:he-did-not}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



\citeA{yoshida-etal-2023-revisiting} demonstrated that by using a modern parser, the scope resolution method based on heuristics for syntax can obtain the same level of performance as the state-of-the-art methods based on end-to-end neural networks.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2.2.2
\subsubsection{NegBERT} \label{subsubsec:negbert}
The other method that we adopt is called NegBERT, which was proposed by \citeA{khandelwal-sawant-2020-negbert}.
This method applies the transfer learning approach of PLMs to a negation scope resolution task.
As a base model, this method uses BERT \cite{devlin-etal-2019-bert} with a classification layer on top of it.
NegBERT can be obtained by fine-tuning this model on a negation-annotated corpus.
This method models the negation scope resolution task as an end-to-end binary sequence labeling task. 
It encodes the sentence along with the position and the type of a negation cue and then, the model outputs whether each word in the sentence is within the negation scope.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3
\section{Negation Scope Conversion} \label{sec:conversion}
This section proposes a method for automatically converting the negation scopes of BioScope\footnote{As in the previous studies, we use the two sub-corpora of BioScope, Abstract and FullPaper, because the original texts of the other sub-corpus (Clinical) are not publicly available.} and SFU to those of Sherlock.
As explained in Section~\ref{subsec:neg-corpora}, most annotation schemes of the negation scope in SFU follow those of BioScope.
Therefore, we regard the scope annotations of BioScope and SFU as almost identical and treat them in the same way.\footnote{Unlike SFU and Sherlock, the scope annotation of BioScope includes the cue. To handle this format difference, we remove cues from scopes of BioScope as a pre-processing.} We refer to the combination of BioScope and SFU as B\&S. Below, we represent a negation scope as a set of indices, each indicating the position of a word in a sentence.

The key idea of our method is to convert scopes effectively by utilizing the existing scope annotations in B\&S.
This idea is based on the following hypotheses, which were developed by observing the training data and the annotation guidelines of the three corpora:
%%%%%
\begin{enumerate}[label=\arabic*.]
  \item The right parts of the scope to the cue can be regarded as almost identical for B\&S and Sherlock.
  \item The middle parts of the scope in the cue can be regarded as almost identical for B\&S and Sherlock.
  \item The left parts of the scope to the cue can be regarded as almost identical for B\&S and Sherlock if the scope of B\&S covers the left of the cue (e.g., Sentence~\ref{snt:bio-passive} is the case).
\end{enumerate}
%%%%%
Based on these hypotheses, we established the following strategy for conversion:
%%%%%
\begin{itemize}
  \item We divide the negation scope into four parts: the left part (denoted as $S_{\mathrm{left}}$), the middle part ($S_{\mathrm{mid}}$), the right part ($S_{\mathrm{right}}$), and the internal part of the cue ($S_{\mathrm{cue}}$).
  \item For $S_{\mathrm{mid}}$ and $S_{\mathrm{right}}$, we utilize the existing scope annotations in B\&S, following Hypotheses~1 and 2.
  \item For $S_{\mathrm{left}}$, we utilize the existing scope annotations in B\&S if the scope annotation in B\&S covered the left part of the cue, following Hypothesis~3. If the scope annotation in B\&S did not cover the left part of the cue, we obtain the left part of the scope using the negation scope resolution method.
\end{itemize}
%%%%%

Below is the formal definition of our conversion method that obtains the converted scope $S_{\mathrm{converted}}$:
%%%%%
\[
  S_{\mathrm{converted}} = S_{\mathrm{left}} \cup S_{\mathrm{cue}}\cup S_{\mathrm{mid}}  \cup S_{\mathrm{right}}
\]
%%%%%
where
%%%%%
\begin{align*}
&S_{\mathrm{left}} = 
\begin{cases}
  \mathrm{L}_c(S_{\mathrm{B\&S}}) & (\mathrm{L}_c(S_{\mathrm{B\&S}})\neq \emptyset) \\
  \mathrm{L}_c(S_{\mathrm{res}}) & (\mathrm{L}_c(S_{\mathrm{B\&S}})=\emptyset),
\end{cases}
\\
&S_{\mathrm{mid}} = \mathrm{M}_c(S_{\mathrm{B\&S}}), 
\\
&S_{\mathrm{right}} = \mathrm{R}_c(S_{\mathrm{B\&S}}).
\end{align*}
%%%%%
Here, $S_{\mathrm{B\&S}}$ and $S_{\mathrm{res}}$ represent the scopes of B\&S and the results of the scope resolution method, respectively.
$c$ is a negation cue.
$S_{\mathrm{cue}}$ denotes the internal structure of the cue and is defined in the next section.
$\mathrm{L}_c(S)$, $\mathrm{M}_c(S)$, and $\mathrm{R}_c(S)$ are the left, middle,\footnote{$\mathrm{M}_c(S)$ exists only when the negation cue consists of multiple words and the words are discontinuous (e.g., \textbf{neither}, \textbf{nor}\dots).} and right parts of the scopes, respectively, and are defined as follows:
%%%%%
\begin{align*}
  \mathrm{L}_c(S)&=\{i\in S\ |\ i < c_l\},\\
  \mathrm{M}_c(S)&=\{i\in S\ |\ c_l < i < c_r\},\\
  \mathrm{R}_c(S)&=\{i\in S\ |\ c_r< i\} .
\end{align*}
%%%%%
Here, $c_l$ and $c_r$ denote indices for the leftmost and rightmost words of the negation cues, respectively.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.1
\subsection{Affixal and Contracted Cues} \label{subsec:aff-cont-cues}
Sherlock distinguishes the cue and its scope in the word for affixal cues (e.g., \textbf{un}\underline{usual}) and contracted cues (e.g., \underline{do}\textbf{n't}), whereas B\&S treats the entire word as the cue.
We distinguish affixal and contracted cues in B\&S based on simple pattern matching.
If B\&S's cue $c$ has an affix in $\mathrm{V}_{\mathrm{aff}}$ or a suffix in $\mathrm{V}_{\mathrm{cont}}$, $S_{\mathrm{cue}}$ is a singleton set consisting of the result by removing the affix from the cue.
Based on the annotation guidelines of Sherlock \cite{morante-etal-2011-guideline} and the training data, we define $\mathrm{V}_{\mathrm{aff}}$ and $\mathrm{V}_{\mathrm{cont}}$ as follows: 
%%%%%
\begin{align*}
  \mathrm{V}_{\mathrm{aff}} = \{\mathrm{dis, im, in, ir, un, less}\}\ ,\ 
  \mathrm{V}_{\mathrm{cont}} = \{\mathrm{n't, not}\}\ .
\end{align*}
%%%%%
Figure~\ref{fig:conv-example-1} shows an example of a scope conversion that includes a contracted cue.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F2
\begin{figure}[b]
\begin{center}
\includegraphics{32-1ia11f2.eps}
\end{center}
\caption{Example of scope conversion where $\mathrm{L}_c(S_{\mathrm{B\&S}}) \neq \emptyset$ and a contracted cue is included.}
\label{fig:conv-example-1}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F3
\begin{figure}[b]
\begin{center}
\includegraphics{32-1ia11f3.eps}
\end{center}
\caption{Example of scope conversion where $\mathrm{L}_c(S_{\mathrm{B\&S}}) = \emptyset$.}
\label{fig:conv-example-2}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.2
\subsection{Negation Scope Resolution} \label{subsec:nsr}
To obtain $S_{\mathrm{res}}$, we can adopt any scope resolution method. 
In the experiments reported later, we chose to use the methods described in Section~\ref{subsec:neg-scope-resolution}. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F4
\begin{figure}[t]
\begin{center}
\includegraphics{32-1ia11f4.eps}
\end{center}
\hangcaption{Parse tree of the sentence ``Surprisingly, the restaurant failed to receive positive reviews.''. Underlined constituents denote scope candidates. Enclosed parts are removed in the adjustment step.}
\label{tree:failed}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F5
\begin{figure}[t]
\begin{center}
\includegraphics{32-1ia11f5.eps}
\end{center}
\hangcaption{Path pattern rules. Each row represents one rule, shown in the order in which they are applied. X/Y means Y is the parent of X, X//Y denotes Y is an ancestor of X, and X\textbackslash Y implies Y is a child of X. The symbol * matches any sequence of zero or more characters, namely, VB* denotes any string starting with VB. The underlined rules are those that we add in this study.}
\label{fig:path-patterns}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


For the method proposed by \citeA{yoshida-etal-2023-revisiting}, we modify Steps~2 and 3 explained in Section~\ref{subsubsec:syntax-based-nsr}.
The modifications adapt to the domains of BioScope and SFU.
Below, we explain our modifications using the example shown in Figure~\ref{fig:conv-example-2} and the parse tree shown in Figure~\ref{tree:failed}.
In Step~2, we add new path pattern rules to Yoshida et al.'s rules.
The additional rules are based on a preliminary experiment using the training data, where we observed that Yoshida et al.'s rules do not cover verb and noun negation cues such as ``lack.''
Figure~\ref{fig:path-patterns} shows the new rule sets.\footnote{As supplementary information, we show the frequency of each rule's application on the training sets in Appendix~\ref{sec:appendix}.} In the example shown in Figure~\ref{fig:conv-example-2}, the rule set selects the constituent S that dominates the entire sentence, with the additional rule \verb|VB*//S| activated. If none of the rules are activated, we use \textit{default scope} proposed by Read et al. For the details on \textit{default scope}, see \citeA{read-etal-2012-uio1}. For Step~3, we modify Yoshida et al.'s scope adjustment rules. We observed that some of Yoshida et al.'s rules specialize in the literary domain, such as Sherlock.
For example, noun phrases delimited by punctuations, which are commonly found in dialogues in literary stories, are often excluded from the scope.
Based on this, Yoshida et al.'s rule removes punctuation-delimited noun phrases from the scope.
However, in the biological and review domains, noun phrases delimited by punctuations are frequently included within the scope.
To address such a difference in domains, we use only a subset of Yoshida et al.'s rules that can apply to general text.
To be specific, our method performs the following adjustments:
%%%%%
\begin{itemize}
  \item Remove constituent-initial punctuations, RB, CC, UH, ADVP, INTJ, or SBAR.
  \item Remove constituent-initial PP that is delimited by a punctuation.
  \item Remove punctuation-delimited ADVP or INTJ.
  \item Remove CC and previous conjuncts if the cue is in a conjoined phrase.
\end{itemize}
%%%%%
In the example shown in Figure~\ref{fig:conv-example-2}, the rules remove ``surprisingly'' and the comma following it.

As seen in the examples shown in Figures~\ref{fig:conv-example-1} and \ref{fig:conv-example-2}, we can accurately convert the scope of B\&S to that of Sherlock.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4
\section{Evaluation of the Accuracy of the Conversion Method} \label{sec:eval-conversion}
We evaluated how accurately the proposed method converts the scopes of B\&S to those of Sherlock.
Section~\ref{subsec:eval-settings} describes the evaluation metrics used to evaluate the accuracy of the conversion method.
For the evaluation, we conducted an experiment as follows: 
First, we obtained a dataset by manually annotating the negation scopes to a tiny portion of B\&S, namely, the test and validation sets of B\&S, following the annotation guidelines of the Sherlock dataset (Section~\ref{subsec:manual-annotation}).
Using this dataset, we evaluated our method based on the metrics (Section~\ref{subsec:eval-results}).
We also qualitatively analyzed the results by performing a manual error analysis on the validation set (Section~\ref{subsec:error-analysis}).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.1
\subsection{Evaluation Metrics}  \label{subsec:eval-settings}
This section describes the evaluation metrics used to evaluate the accuracy of the conversion method.
In the evaluation, we focus specifically on the important hypotheses, which the method adopts: 
(i) $\mathrm{R}_c(S_{\mathrm{B\&S}})$ and $\mathrm{R}_c(S_{\mathrm{SH}})$ can be regarded as almost identical,
(ii) $\mathrm{M}_c(S_{\mathrm{B\&S}})$ and $\mathrm{M}_c(S_{\mathrm{SH}})$ can be regarded as almost identical, and
(iii) $\mathrm{L}_c(S_{\mathrm{B\&S}})$ and $\mathrm{L}_c(S_{\mathrm{SH}})$ can be regarded as almost identical if $\mathrm{L}_c(S_{\mathrm{B\&S}}) \neq \emptyset$.
The evaluation metric \textbf{\textit{hypo-token}} considering this point is summarized as follows:
%%%%%
\begin{itemize}
  \item \textit{Hypo-token} checks whether the converted scopes and the ground-truth match at the token level.
  \item \textit{Hypo-token} evaluates only the tokens that belong to the scopes that were assumed to be almost identical between B\&S and Sherlock.
        Specifically, the tokens that belong to the following parts are evaluated:
    \begin{itemize}
      \item $\mathrm{R}_c(S_{\mathrm{converted}})$ and $\mathrm{M}_c(S_{\mathrm{converted}})$.
      \item $\mathrm{L}_c(S_{\mathrm{converted}})$ if $\mathrm{L}_c(S_{\mathrm{B\&S}}) \neq \emptyset$.
    \end{itemize}
\end{itemize}
%%%%%
We adopted \textit{hypo-token} as the primary evaluation metric.

In addition to \textit{hypo-token}, we used the following two metrics that are commonly used for the evaluation of negation scope resolution:
%%%%%
\begin{itemize}
  \item \textbf{\textit{Scope-strict-match}}\footnote{This metric is commonly referred to as \textit{scope(s)} or \textit{scope-level}; however, in this paper, we refer to it as \textit{scope-strict-match} for clarity.}: This metric checks the scope labels for all the tokens inside the scope for each negation cue; TP, TN, FP, or FN is assigned to each scope.
      To consider the scope as TP, all tokens inside the scope of a negation cue must match between the target and the ground-truth.
      Partial matches are considered as FN. 
      A high score in this metric indicates the high capability of the method (or model) to convert (or predict) scopes perfectly.
  \item \textbf{\textit{Scope-token}}: This metric checks the scope label of each token; TP, TN, FP, or FN is assigned to each token.
      A high score on this metric indicates an overall small gap between the two scopes.
\end{itemize}
%%%%%
For all the metrics, the precision, recall, and $\mathrm{F_1}$ measures are calculated.
If a sentence contains more than one negation cue, evaluation is conducted for each cue separately.
Note that punctuation tokens are also included in the evaluation.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.2
\subsection{Manual Scope Annotation} \label{subsec:manual-annotation}
Since the proposed method converts the scopes of B\&S to those of Sherlock, the evaluation requires B\&S data annotated with negation scopes following the Sherlock standard.
We constructed a dataset by manually annotating a tiny portion of B\&S, namely the test and validation sets\footnote{The train-val-test split is identical to that of \citeA{truong-etal-2022-improving}. We referred to \url{https://github.com/joey234/negation-focused-pretraining}.} with negation scopes following the annotation guidelines of Sherlock \cite{morante-etal-2011-guideline}.
The test set was used to measure the accuracy of the conversion method and the validation set was used to measure the inter-annotator agreement (IAA) and manual error analysis. 
We applied the NLTK \cite{bird-loper-2004-nltk} tokenizer to BioScope as a pre-processing step because SFU and Sherlock are tokenized based on the Penn Treebank \cite{marcus-etal-1993-building} standard, whereas BioScope is not.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.2.1
\subsubsection{Annotation Details} \label{subsubsec:annotation-details}
The annotation was performed by a worker (not one of the authors) who was proficient in the NLP annotation tasks.
We asked the worker to annotate the negation scopes following the Sherlock standard, given the sentences and negation cues of B\&S.
With regard to the negation cues, we instructed the worker to use cues as they are; however, one exception exists: 
The worker extracted substrings from the negation cues of B\&S, if they are affixal or contracted cues in the Sherlock standard.
This is intended to handle the difference in the annotations for affixal and contracted cues between B\&S and Sherlock as mentioned in Section~\ref{subsec:aff-cont-cues}.

The annotation process is as follows:
%%%%%
\begin{enumerate}[label=\arabic*.]
  \item The authors provided the worker with instructions for the annotation task and the annotation guidelines of Sherlock \cite{morante-etal-2011-guideline}.
        A small set of instances\footnote{
          The instances were extracted from the training set of B\&S.
        } and their correct annotations were also provided to facilitate the worker's understanding of the task.
        Using these materials, the worker understood the details of the annotation process.
  \item The authors provided the worker with the test and validation sets of B\&S and the worker performed the annotation task.
        Each instance contained a sentence and a negation cue.
        Note that the worker was not provided with the scope annotation of B\&S.
        It should also be noted that the worker and researchers did not discuss the content of scope annotations.
\end{enumerate}
%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.2.2
\subsubsection{Annotation Quality Assessment} \label{subsubsec:iaa}
To assess the quality of annotations by the worker, one of the authors annotated a subset of the validation sets: 
a total of 100 sentences, consisting of 50 randomly extracted sentences from each corpus (out of 291 sentences in the BioScope validation set and 468 of SFU).
For the 100 sentences, we calculated the IAA between the worker and the author.
Below, the scope annotations by the worker and the author are called $\mathrm{SA_{worker}}$ and $\mathrm{SA_{author}}$, respectively.


We calculated the IAA in terms of precision, recall, and $\mathrm{F_1}$ measure following \citeA{morante-daelemans-2012-conandoyle}.
For the calculation, $\mathrm{SA_{author}}$ was assessed against $\mathrm{SA_{worker}}$.
IAA was calculated by \textit{scope-strict-match} and \textit{scope-token}, both of which are explained in Section~\ref{subsec:eval-settings}.


Tables~\ref{tab:iaa-scope} and \ref{tab:iaa-token} show the IAA scores between $\mathrm{SA_{worker}}$ and $\mathrm{SA_{author}}$.
For comparison, we present the IAA scores in the scope annotations of Sherlock in Table~\ref{tab:sherlock}, obtained by \citeA{morante-daelemans-2012-conandoyle}.
At the metric of \textit{scope-strict-match}, the IAA score between $\mathrm{SA_{worker}}$ and $\mathrm{SA_{author}}$ had a precision of 100\% and a recall of approximately 75\% (Table~\ref{tab:iaa-scope}).
This implies approximately 75\% of the scope annotations between the worker and the author match perfectly, whereas approximately 25\% differ partially, considering that TP represents perfect matches and FN represents partial matches in the metric of \textit{scope-strict-match}.
Considering the recall of 79.08\% and 69.43\% in the Sherlock for the \textit{cue-scope-strict} metric, the scores between $\mathrm{SA_{worker}}$ and $\mathrm{SA_{author}}$ can be comparable, whereas we cannot directly compare the \textit{cue-scope-strict} and \textit{scope-strict} metrics.
For the \textit{scope-token} metric, the IAA score between $\mathrm{SA_{worker}}$ and $\mathrm{SA_{author}}$ has an $\mathrm{F_1}$ measure of approximately 90\% (Table~\ref{tab:iaa-token}), which is comparable to the scores for the Sherlock: 91.53\% and 88.04\% (Table~\ref{tab:sherlock}).\footnote{From Tables~\ref{tab:iaa-token} and \ref{tab:sherlock}, we can observe that for the metric of \textit{scope-token}, the IAA scores in this study have higher precision than recall, whereas those in the Sherlock dataset have higher recall than precision. This difference is due to the selection of the ground-truth in the IAA calculation. The precision and recall values vary depending on which annotator's annotation is used as the ground-truth for the agreement calculation.} Considering these points, we concluded that the quality of $\mathrm{SA_{worker}}$ was acceptable and used it to evaluate the conversion method.




%%%%%%%%%%%%%%%%%%%%%%%%T2
\begin{table}[t]
\input{11table02.tex}
\hangcaption{IAA in \textit{scope-strict-match}: scope annotations by the author was assessed against those by the worker.}
\label{tab:iaa-scope}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%T3
\begin{table}[t]
\input{11table03.tex}
\hangcaption{IAA in \textit{scope-token}: scope annotations by the author was assessed against those by the worker.}
\label{tab:iaa-token}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T4
\begin{table}[t]
\input{11table04.tex}
\hangcaption{IAA in the Sherlock dataset. The scores show the IAA between two annotators, quoted from \protect\citeA{morante-daelemans-2012-conandoyle}. The annotations of one annotator were assessed against those of the other annotator (official annotation of Sherlock). Since Sherlock annotates both negation cues and their scopes, the IAA metrics are partially different between \protect\citeA{morante-daelemans-2012-conandoyle}, and those of this paper. \textit{Cue} evaluates whether tokens of negation cues match; \textit{cue-scope-strict} (referred to as \textit{scopes} in the original paper) evaluates whether negation cues and their scopes match strictly; \textit{scope-token} is the same metric as this paper. Note that \textit{cue-scope-strict} is a stricter metric than \textit{scope-strict} in this paper.}
\label{tab:sherlock}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.3
\subsection{Experiment} \label{subsec:eval-results}
We experimentally evaluated the accuracy of the proposed conversion method using the dataset described in Section~\ref{subsec:manual-annotation}.
We applied the conversion method to the test sets of B\&S to obtain its converted version, which we refer to as $\mathrm{SA_{converted}}$.
$\mathrm{SA_{converted}}$ was evaluated against $\mathrm{SA_{worker}}$.
For the scope resolution process in the conversion method, we used two methods: \citeA{yoshida-etal-2023-revisiting}'s one with modifications and \citeA{khandelwal-sawant-2020-negbert}'s one {\linebreak}(NegBERT), both of which are explained in Section~\ref{subsec:neg-scope-resolution}.
The parsing process in the method proposed by Yoshida et al.'s was performed using the Berkeley Neural Parser \cite{kitaev-klein-2018-constituency,kitaev-etal-2019-multilingual} with BERT \cite{devlin-etal-2019-bert}.
For obtaining NegBERT, we fine-tuned BERT on the Sherlock dataset for negation scope resolution using the code and the hyperparameters provided by \citeA{truong-etal-2022-improving}.\footnote{We referred to \url{https://github.com/joey234/negation-focused-pretraining}. This code includes the re-implementation of \citeA{khandelwal-sawant-2020-negbert}, which we used for obtaining NegBERT.}

The accuracy of the conversion method was evaluated based on the three metrics described in Section~\ref{subsec:eval-settings}.
As a comparison method, we directly used the scope resolution methods, which simply predict the scope labels for all tokens without using any scope labels of B\&S.
This can be considered as an ablation of our conversion.

Table~\ref{tab:eval-hypo-token} shows the evaluation results of the \textit{hypo-token} metric.
For this metric, the proposed conversion method achieved an $\mathrm{F_1}$ measures of 89.95\% and 82.08\% for BioScope and SFU, respectively.
The method achieved relatively high precision scores for both BioScope and SFU.
Particularly, the score in BioScope exceeded 98\%.
This indicates that almost all tokens within the scopes of BioScope are also included in Sherlock scopes for the parts where we assumed that the scopes are almost identical between B\&S and Sherlock.
The precision for SFU also exceeds 90\%, confirming that most of the tokens within the scopes of SFU are also included in those of Sherlock for the mentioned parts.



%%%%%%%%%%%%%%%%%%%%%%%%%T5
\begin{table}[t]
\input{11table05.tex}
\hangcaption{ Evaluation results for \textit{hypo-token}: $\mathrm{SA_{converted}}$ was evaluated against $\mathrm{SA_{worker}}$. Along with the overall scores (All), scores for the left, middle, and right parts of the scopes ($\mathrm{L}_c(S_{\mathrm{converted}})$, $\mathrm{M}_c(S_{\mathrm{converted}})$, $\mathrm{R}_c(S_{\mathrm{converted}})$) are also provided. Note that for \textit{hypo-token}, the score for $\mathrm{L}_c(S_{\mathrm{converted}})$ is counted only if $\mathrm{L}_c(S_{\mathrm{B\&S}}) \neq \emptyset$.}
\label{tab:eval-hypo-token}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%



For recall, in contrast, the method shows relatively lower scores: 82.59\% for BioScope and 75.00\% for SFU.
This shows that approximately 17.41\% of the tokens in BioScope and 25.00\% in SFU are not included in the scopes of B\&S but are included in those of Sherlock for the parts where we assumed that scopes are almost identical between B\&S and Sherlock.
To identify the pattern of these cases, which our hypotheses do not cover, we perform a manual error analysis and provide a detailed discussion in Section~\ref{subsec:error-analysis}.

Examining each part, the proportion of target tokens in $\mathrm{L}_c(S_{\mathrm{converted}})$ is relatively low (735 out of 11575).
This is because, as mentioned in Section~\ref{subsec:neg-corpora}, scope annotations in B\&S cover the left part of the cue only in exceptional cases, such as passive sentences.
In these cases, the proposed method achieved an $\mathrm{F_1}$ measure of 90.21\% in BioScope.
With regard to SFU, target tokens of $\mathrm{L}_c(S_{\mathrm{converted}})$ are only 11, which implies that exceptional cases rarely appear in the test set of SFU.
The target tokens of $\mathrm{M}_c(S_{\mathrm{converted}})$ are also low because the middle part of the scope exists only in limited negation cues such as \textit{neither, nor}.
In this experiment, seven target tokens were present, all of which were accurately converted.
We observe that the majority of the targets for \textit{hypo-token} are in $\mathrm{R}_c(S_{\mathrm{converted}})$, that is, $\mathrm{R}_c(S_{\mathrm{converted}})$ is the most important in \textit{hypo-token} to achieve a high accuracy of conversion.
For this part, the performance of the conversion method was 89.87\% for BioScope and 82.10\% for SFU in terms of the $\mathrm{F_1}$ measure.
This relatively low score for SFU affects the overall score of SFU for \textit{hypo-token}, making it relatively lower than that for BioScope.
The reasons for this are discussed in Section~\ref{subsec:error-analysis}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T6
\begin{table}[t]
\input{11table06.tex}
\hangcaption{Evaluation results for \textit{scope-strict-match}: $\mathrm{SA_{converted}}$ was evaluated against $\mathrm{SA_{worker}}$. Note that we applied the modifications explained in Section~\ref{subsec:nsr} to the method proposed by \protect\citeA{yoshida-etal-2023-revisiting}.}
\label{tab:eval-scope-strict}
{\vskip 10pt}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T7
\begin{table}[t]
\input{11table07.tex}
\hangcaption{Evaluation results for \textit{scope-token}: $\mathrm{SA_{converted}}$ was evaluated against $\mathrm{SA_{worker}}$. Note that we applied the modifications explained in Section~\ref{subsec:nsr} to the method proposed by \protect\citeA{yoshida-etal-2023-revisiting}.}
\label{tab:eval-scope-token}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%




Tables~\ref{tab:eval-scope-strict} and \ref{tab:eval-scope-token} show the evaluation results for the metrics of \textit{scope-strict-match} and \textit{scope-token}, respectively.
For \textit{scope-strict-match}, the scope conversion methods, both with \citeA{yoshida-etal-2023-revisiting} and NegBERT, outperformed the scope resolution methods.
In particular, scope conversion with \citeA{yoshida-etal-2023-revisiting} achieved significantly higher scores than scope resolution by \citeA{yoshida-etal-2023-revisiting}.
This performance difference was mainly due to the gap in recall scores: 57.09\% for conversion and 37.38\% for resolution.
A low recall score for \textit{scope-strict-match} implies the number of partial matches is high compared with that of perfect matches.
Therefore, the results suggest that our utilization of the B\&S scopes is effective for scope conversion in increasing the number of perfect scope matches.
In terms of \textit{scope-token}, in contrast, the conversion methods showed slightly lower scores than the resolution methods for the $\mathrm{F_1}$ measure for both \citeA{yoshida-etal-2023-revisiting} and NegBERT.
The scores of the conversion methods show relatively higher precision and lower recall, which is consistent with the results for \textit{hypo-token}, shown in Table~\ref{tab:eval-hypo-token}.
To identify the pattern of error cases, we report manual error analysis in the next section.
As mentioned in Section~\ref{subsec:eval-settings}, a \textit{hypo-token} is a more appropriate metric than \textit{scope-token} for evaluating the proposed conversion method.

Table~\ref{tab:eval-resolved-token} presents the evaluation results for the tokens predicted by the scope resolution method, that is, token-level evaluation for the scopes $\mathrm{L}_c(S_{\mathrm{converted}})$ where $\mathrm{L}_c(S_{\mathrm{B\&S}})=\emptyset$, which is the subset of \textit{scope-token}.
Comparing the scores of \citeA{yoshida-etal-2023-revisiting}'s method and NegBERT, a difference was observed between BioScope and SFU: 
The method proposed by Yoshida et al. achieved higher $\mathrm{F_1}$ measures on BioScope than on SFU, whereas NegBERT achieved higher $\mathrm{F_1}$ measures on SFU than on BioScope.
This difference can be explained by the influence of both the domain of the corpus and the characteristics of the scope resolution methods.
BioScope consists of formal, grammatically structured biomedical texts that allow the syntax-based method proposed by Yoshida et al. to resolve scopes more accurately than NegBERT, even with complex syntactic structures.
In contrast, SFU consists of review texts that are often informal and non-grammatical, which makes it difficult for the syntax-based method to resolve scopes accurately.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T8
\begin{table}[b]
{\vskip 10pt}
\input{11table08.tex}
\hangcaption{Evaluation results for the tokens predicted by the scope resolution methods. This is the subset of \textit{scope-token} where $\mathrm{L}_c(S_{\mathrm{converted}})$ if $\mathrm{L}_c(S_{\mathrm{B\&S}})=\emptyset$. Note that we applied the modifications explained in Section~\ref{subsec:nsr} to the method proposed by \protect\citeA{yoshida-etal-2023-revisiting}.}
\label{tab:eval-resolved-token}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



Referring to Tables~\ref{tab:eval-hypo-token} and \ref{tab:eval-resolved-token}, we discuss the degree of the importance of the two factors in our method: the correctness of the hypotheses and the accuracy of the scope resolution method.
Comparing the number of target tokens, the number of \textit{hypo-token} is greater than that of tokens predicted by the scope resolution methods (11575 vs. 9509).
This indicates that the hypotheses play a more important role than the scope resolution method in enhancing the accuracy of the proposed conversion method.
However, since no significant difference exists in the number of target tokens, the performance of a negation scope resolution method is also important for the conversion method.
Given that the performances of the scope resolution methods is relatively low (74.53\%/69.93\% for $\mathrm{F_1}$ measure) compared with the score in \textit{hypo-token} (85.63\% for $\mathrm{F_1}$ measure), we need a negation scope resolution method that can achieve higher accuracy across different domains in order to improve the accuracy of the conversion method.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.4
\subsection{Manual Error Analysis} \label{subsec:error-analysis}
We performed manual error analysis of the conversion method on the validation set of B\&S, using the output of Scope Conversion with \citeA{yoshida-etal-2023-revisiting}, which outperformed Scope Conversion with NegBERT.
We collected the instances in which the conversion output $S_{\mathrm{converted}}$ did not match the scope annotation $S_{\mathrm{worker}}$, namely, the instances where scope labels of $S_{\mathrm{converted}}$ and $S_{\mathrm{worker}}$ differed in at least one token for the negation cue.
Here, $S_{\mathrm{worker}}$ represents the scope annotation of the worker in each instance.
The collected instances on the validation sets were 349 in total (98 in the BioScope Abstract sub-corpus, 32 in the BioScope FullPaper sub-corpus and 219 in SFU).
Out of these, we extracted 230 instances (98 in BioScope Abstract, 32 in BioScope FullPaper and the initial 100 in SFU) as candidates for analysis and re-checked the scope annotation $S_{\mathrm{worker}}$.
We observed obvious mistakes in $S_{\mathrm{worker}}$ in 15 instances (8 in BioScope Abstract and 7 in SFU).
We corrected these mistakes to remove their effects.
If the correction led to $S_{\mathrm{converted}}$ and $S_{\mathrm{worker}}$ matching, such instances were excluded.
Finally, we manually analyzed 215 instances (90 in BioScope Abstract, 32 in BioScope FullPaper, and 93 in SFU).
Conversion errors were observed in the left parts $\mathrm{L}(S_\mathrm{converted})$, or right parts $\mathrm{R}(S_\mathrm{converted})$ of the scopes; we counted the error patterns for $\mathrm{L}(S_\mathrm{converted})$ and $\mathrm{R}(S_\mathrm{converted})$ separately.

We classified the causes of the errors into the following categories:
%%%%%
\begin{itemize}
  \item $\mathrm{E_{hypotheses}}$: The hypotheses of the conversion method are invalid. In other words, the scope that was assumed to be almost identical between B\&S and Sherlock is actually different.
  \item $\mathrm{E_{resolution}}$: The prediction of the scope resolution method, used for the parts where the scopes of B\&S and Sherlock were not assumed to be almost identical, is incorrect.
  \item $\mathrm{E_{B\&S}}$: The scope annotation assigned to B\&S is incorrect in the B\&S standard.
\end{itemize}
%%%%%

Table~\ref{tab:error-analysis} shows the classification statistics.
The instances classified as $\mathrm{E_{hypotheses}}$ are the most important ones for investigating the proposed conversion method.
Some instances of errors in SFU classified as $\mathrm{E_{hypotheses}}$ contained informal or ungrammatical expressions.
The following instance is such an example:
%%%%%
\begin{itemize}[itemsep=5pt]
  \item $S_\mathrm{converted}$ \\
  \textbf{\textit{No}} \textit{\underline{doubt, beautiful}.}
  \item $S_\mathrm{worker}$ \\
  \textbf{\textit{No}} \textit{\underline{doubt}, beautiful.}
  \item $S_\mathrm{B\&S}$ \\
  \textbf{\textit{No}} \textit{\underline{doubt, beautiful}.}
\end{itemize}
%%%%%
In this example, $S_\mathrm{converted}$ and $S_\mathrm{worker}$ differ in the right part of the cue: $S_\mathrm{converted}$ included the adjective \textit{beautiful} in the scope but $S_\mathrm{worker}$ did not.
This instance includes informal expressions and is not a typical complete sentence with a subject and a verb.
Although Sherlock provides detailed annotation guidelines to handle such incomplete sentences, the guidelines of BioScope and SFU do not mention such cases and likely rely on annotator judgment.
This difference in the guidelines can cause different annotations between B\&S and Sherlock for informal or incomplete sentences, which implies that our hypotheses do not hold in such cases.
Since SFU consists of review-domain text, informal or ungrammatical expressions frequently appear as seen in the example above.
In contrast, BioScope consists of formal biomedical texts, where such expressions are less frequent.
Consequently, the lower \textit{hypo-token} score in SFU compared with BioScope seen in Table~\ref{tab:eval-hypo-token} is partly due to the cases in which the hypotheses do not hold in informal or ungrammatical sentences.
This is a limitation of the proposed method, which should be addressed in future work to enhance the conversion accuracy.



%%%%%%%%%%%%%%%%%%%%%%%%%%%T9
\addtocounter{table}{-1}
\begin{table}[t]
\input{11table09.tex}
\hangcaption{Number of error instances by causes for $\mathrm{L}(S_\mathrm{converted})$ and $\mathrm{R}(S_\mathrm{converted})$. The bracketed numbers in (b) represent the number of instances classified as $\mathrm{E_{hypotheses}}$ that contain clauses or phrases expressing reason, which are related to the variations in the scope annotation following the guidelines of Sherlock. Note that the unbracketed value in each column includes the bracketed value.}
\label{tab:error-analysis}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%



Moreover, $\mathrm{E_{hypotheses}}$ has a frequent pattern in which a sentence contains adverbial clauses or phrases that express reason (such as a \textit{because} clause).
This pattern is explained using the following example: 
%%%%%
\begin{itemize}[itemsep=5pt]
  \item $S_\mathrm{converted}$ \\
  \textit{\underline{This shift is apparently}} \textbf{\textit{not}} \textit{\underline{caused by a recruitment phenomenon}, because in FCS+\\ culture, the total number of colonies is not significantly modified by RA addition.}
  \item $S_\mathrm{worker}$ \\
  \textit{\underline{This shift is apparently}} \textbf{\textit{not}} \textit{\underline{caused by a recruitment phenomenon, because in FCS+} \\ \underline{culture, the total number of colonies is not significantly modified by RA addition}.}
  \item $S_\mathrm{B\&S}$ \\
  \textit{\underline{This shift is apparently}} \textbf{\textit{not}} \textit{\underline{caused by a recruitment phenomenon}, because in FCS+\\ culture, the total number of colonies is not significantly modified by RA addition.}
\end{itemize}
%%%%%
In this instance, $S_\mathrm{converted}$ and $S_\mathrm{worker}$ differ in the right part of the cue, where the method directly utilized $S_\mathrm{B\&S}$ as the output.
This difference occurs in the \textit{because}-clause: $S_\mathrm{worker}$ included the clause as the scope but $S_\mathrm{converted}$ did not.
During the analysis, we found that whether or not a \textit{because}-clause is included in the scope depends on the annotator's judgment, even following the guidelines of Sherlock: 
According to the principle that the scope of the cue \textbf{\textit{not}} that negates a main verb includes subordinate clauses, the \textit{because}-clause should be included in the scope.
However, if an annotator applies \textit{it is not the case that} paraphrase test\footnote{For the detail of the \textit{it is not the case that} paraphrase test, see \citeA{morante-etal-2011-guideline}.} provided in the guidelines, which can be applied when doubts arise in annotation, the \textit{because}-clause can be excluded from the scope.
In fact, Sherlock contains instances with and without the \textit{because}-clause annotated as part of the scope.
The error pattern regarding clauses or phrases that express reason was frequently observed, suggesting that the relatively low recall scores in the token-level metrics (\textit{hypo-token} and \textit{scope-token}), as shown in Section~\ref{subsec:eval-results}, were partly due to these variations in scope annotation.
Handling these annotation variations is difficult when designing a conversion method.



{\vskip -5pt}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5
\section{Negation Scope Resolution with a Unified Dataset} \label{sec:data-aug}
{\vskip -5pt}
We conducted experiments on negation scope resolution to evaluate the effectiveness of the proposed method in terms of scaling up the dataset through our scope conversion.
By adapting our conversion method to B\&S, we obtained a converted version of B\&S, namely, B\&S whose scope annotation is almost identical to that of Sherlock.
We merged the training sets of Sherlock and the converted version of B\&S into a unified dataset.
Using the unified dataset, we fine-tuned PLM-based models and performed negation scope resolution.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.1
\subsection{Experimental Settings} \label{subsec:exp-settings}
We fine-tuned BERT \cite{devlin-etal-2019-bert} and RoBERTa \cite{liu2019roberta}\footnote{We used \textit{bert-base-uncased} for BERT and \textit{roberta-base} for RoBERTa, both of which are released on Hugging Face (\url{https://huggingface.co/}).}
for negation scope resolution using the code of \citeA{truong-etal-2022-improving}.\footnote{We referred to \url{https://github.com/joey234/negation-focused-pretraining}, which includes the re-implementation of \citeA{khandelwal-sawant-2020-negbert}. Note that the method of fine-tuning BERT for negation scope resolution was proposed by \citeA{khandelwal-sawant-2020-negbert}.} The evaluation was performed under the different configurations of fine-tuning: 
%%%%%
\begin{enumerate}[label=(\roman*)]
  \item \textbf{Sherlock only}: using only the training set of Sherlock.
  \item \textbf{Combination}: using the training sets of Sherlock and the unconverted (original) version of B\&S.
  \item \textbf{Resolution + Combination (R + C)}: using the training set of Sherlock along with the training sets of B\&S whose all scope labels were predicted by a scope resolution method.
        We used two different scope resolution methods: \citeA{yoshida-etal-2023-revisiting} with our modifications and NegBERT.
  \item \textbf{Conversion + Combination (C + C)}: using the training sets of Sherlock and the converted version of B\&S, that is, using the unified dataset created using our conversion method.
        For the negation scope resolution process in the conversion method, we used the scope resolution method by \citeA{yoshida-etal-2023-revisiting} with the modifications explained in Section~\ref{subsec:nsr}.
        This is because the performance of scope conversion with \citeA{yoshida-etal-2023-revisiting} outperformed that of scope conversion with NegBERT, as discussed in Section~\ref{subsec:eval-results}.
\end{enumerate}
%%%%%
For the validation and the evaluation of the models, we used the validation and test sets of Sherlock, respectively.
For all the settings, we adopted the same hyperparameters as those used by \citeA{truong-etal-2022-improving}.

We adopted two evaluation metrics, \textit{scope-token} and \textit{scope-strict-match}, both of which are explained in Section~\ref{subsec:eval-settings}.
For both metrics, we calculated the precision, recall, and $\mathrm{F_1}$ measures.
We adopted these metrics because they are the major metrics that have been used in many previous studies since their proposal in *SEM2012 shared task \cite{morante-blanco-2012-sem}.
The use of these metrics enables a fair comparison between the results of this study and those of the previous studies.
Note that there are two differences used in this experiment and *SEM2012 shared task: in this experiment, punctuations are also included in the evaluation and gold cues are given.
These differences stem from the adjustments made to align with the evaluation metrics used in the previous studies on PLM-based negation scope resolution.




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T10
\begin{table*}[b]
{\vskip -10pt}
\input{11table10.tex}
\hangcaption{Results for negation scope resolution with the different configurations of fine-tuning. The scores in our experiments are an average of five runs with different random seeds. Previous studies used only Sherlock for fine-tuning. Values marked in bold and underlined represent the highest and the second-highest scores for each evaluation metric, respectively. \\
$^\ast$ BERT fine-tuned on Sherlock only and RoBERTa on Sherlock only can be regarded as reproductions of \protect\citeA{khandelwal-sawant-2020-negbert}, and \protect\citeA{truong-etal-2022-improving} -- Baseline, respectively. The score difference between the reproduction and the original paper is due to the differences in random seeds.\\
$^{\ast\ast}$~\protect\citeA{yoshida-etal-2023-revisiting} used PLMs only for syntactic parsing.}
\label{tab:results}
\end{table*}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.2
\subsection{Experimental Results} \label{subsec:exp-results}
Table~\ref{tab:results} shows the experimental results.
The results demonstrate that the models trained on the simply combined dataset of the three corpora (Combination) performed worse than those trained on the Sherlock dataset alone (Sherlock only).
In contrast, the performance of the models improved when using the unified dataset (C + C).
These results are observed in both models (BERT and RoBERTa) and in both evaluation metrics (\textit{scope-token} and \textit{scope-strict-match}), supporting the effectiveness of our conversion method in terms of scaling up the dataset.
In addition, the models trained on C + C outperformed those trained on R + C (\citeA{yoshida-etal-2023-revisiting} + Combination/NegBERT + Combination) for both the models and both evaluation metrics.
This result suggests that our strategy of utilizing the existing scope labels in B\&S  is effective for improving the performance of PLM-based negation scope resolution models based on the scaling up of the dataset.

With regard to the \textit{scope-token} metric, the models trained on the unified dataset (C + C) achieved lower precision and higher recall than the baseline, resulting in a higher $\mathrm{F_1}$ measure.
The higher recall of C + C indicates that the models assigned more scope labels by trained on more variants of Sherlock, and produced fewer False Negative predictions.
Since PLM-based methods tend to have lower recall scores in \textit{scope-token} than syntax-based methods (such as \citeA{yoshida-etal-2023-revisiting}'s method), improving the recall score is crucial for improving the performance of PLM-based models.
Therefore, the higher recall of C + C suggests that training on the unified dataset can enhance the performance of negation scope resolution models based on PLMs.

For both metrics, BERT consistently outperformed RoBERTa.
This trend is consistent with the previous studies, showing that BERT achieves better performance in terms of negation scope resolution.
For $\mathrm{F_1}$ measures in \textit{scope-token}, BERT trained on the unified dataset (C + C) outperformed the state-of-the-art models: \citeA{wu-sun-2023-negation}, who proposed the model considering boundary shift loss, and \citeA{truong-etal-2022-improving}, who performed additional pre-training with negation.
This finding demonstrates that the models can achieve high performance with basic fine-tuning on the unified dataset without requiring additional pre-training or using alternative model structures, in terms of \textit{scope-token}.
For \textit{scope-strict-match}, the state-of-the-art models outperformed the results of our experiments.
However, there is a possibility for the models to achieve even higher performance by fine-tuned on our unified dataset, which we leave the exploration for future work.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T11
\begin{table}[b]
\input{11table11.tex}
\hangcaption{Comparison of \textit{scope-token} performance for each type of negation cue between Sherlock only (referred to as SH here) and Conversion + Combination (C + C).The base model is BERT, and performance was measured in the validation set.}
  \label{tab:dev-analysis-token}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T12
\begin{table}[b]
\input{11table12.tex}
\hangcaption{Comparison of \textit{scope-strict-match} performance for each type of negation cue between Sherlock only (referred to as SH here) and Conversion + Combination (C + C). The base model is BERT, and performance was measured in the validation set.}
\label{tab:dev-analysis-scope}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.3
\subsection{Analysis} \label{subsec:exp-analysis}
To further investigate the effectiveness of scaling up the dataset through scope conversion, we conducted an analysis on the validation set of the Sherlock dataset.
We classified the instances into several categories based on the types of negation cues and calculated the model performance for each category in the two settings: Sherlock only and C + C.
The types of negation cues used for classification are as follows:
%%%%%
\begin{itemize}
  \item Single word cue: negation cue that consists of a single word, such as ``\textbf{not}'' and ``\textbf{no}''.
  \item Affixal cue: negation cue that is an affix, such as ``\textbf{im}(possible)'' and ``\textbf{un}(known)''.
  \item Contracted cue: negation cue that is contracted with a verb, such as ``(do)\textbf{n't}''.
  \item Multiword cue: negation cue that consists of multiple words, such as ``\textbf{no longer}''.
\end{itemize}
%%%%%
The PLM model used for the analysis was BERT, which outperformed RoBERTa in the experiments on the test set (Table~\ref{tab:results}).

Tables~\ref{tab:dev-analysis-token} and \ref{tab:dev-analysis-scope} show the results of the analysis for the metrics \textit{scope-token} and \textit{scope-strict-match}, respectively.
In the \textit{scope-strict-match} metric, some categories contain excessively few instances to perform a meaningful analysis; therefore, our analysis focuses on the results of \textit{scope-token}.
The results in Table~\ref{tab:dev-analysis-token} demonstrate that, in terms of $\mathrm{F_1}$ measure, the performance in the categories affixal and multiword cues improved with Conversion + Combination (C + C) compared to Sherlock only (SH) ($+$3.85 and $+$5.23 point, respectively); the performance for single word cues remained nearly unchanged; the results for contracted cues showed a decrease in performance ($-$4.06 point).
Consequently, along with the performance for Sherlock only, our method contributed to enhancing the performance in the categories where Sherlock only had relatively low performance.
We observed the most significant performance gains for affixal cues. 
Since there are various kinds of affixal cues, it is considered that our dataset scaling up leads to performance improvement in this category.
However, the results for contracted cues showed decreased performance in C + C compared with Sherlock only.
A possible reason for this decrease is the relatively poor performance of our conversion method for the scopes of contracted cues.
This is a limitation of the proposed method in terms of scaling up the dataset, which should be addressed in future work.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6
\section{Conclusion} \label{sec:conclusion}
This study proposed a method for automatically converting the negation scopes of B\&S to those of Sherlock, utilizing the existing scope annotations in B\&S.
We conducted an experiment to evaluate the accuracy of the proposed method.
The results demonstrated that our method can convert the scopes of B\&S to those of Sherlock with high accuracy and that our approach of utilizing the correct scopes of B\&S is effective.
Using the proposed method, we merged B\&S and Sherlock into a unified dataset.
To verify the effectiveness of our method in terms of scaling up the dataset, we conducted experiments of negation scope resolution using the unified dataset for fine-tuning PLM-based models.
The experimental results demonstrated that the unified dataset improves the performances of the models, unlike a simple combination of the corpora.
Potential future works include exploring the reverse scope conversion: converting the scopes of Sherlock to those of B\&S.
By automatically identifying and excluding the left parts of the negation scope to the cue(s) in Sherlock that are not annotated in B\&S, we expect to convert the scopes of Sherlock to those of B\&S.
Examining the details of this method will be a topic for future research.
Another future work involves fine-tuning the state-of-the-art negation scope resolution model proposed by \citeA{wu-sun-2023-negation} on our unified dataset, in which we can expect to achieve better performance.



% Start Acknowledgment
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment

This research was partially supported by the Grant-in-Aid for Scientific Research (C) (No. 22K12148) of JSPS.



% Start bibliography
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.7}
\bibliography{11refs}



\appendix
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%A
\section{Application Frequency of Each Path Pattern Rules} \label{sec:appendix}
Table~\ref{tab:rule-freq} shows the application frequency of each path pattern rule in the training sets of BioScope and SFU.


%%%%%%%%%%%%%%%%%%%%%%%%%%T13
\begin{table}[ht]
\input{11table13.tex}
\hangcaption{Application frequency of each path pattern rule (Figure~\ref{fig:path-patterns}) in the training sets of BioScope and SFU. The underlined rules are those that we added in this study. The count was measured per instance (i.e., per negation cue).}
\label{tab:rule-freq}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%



\clearpage
% Start biography
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Asahi Yoshida}{%
  Asahi Yoshida received his bachelor's degree in informatics from Nagoya University in 2023.
  He is currently a master student at Graduate School of Informatics, Nagoya University.
  His research interests include natural language processing, especially neagtion processing in natural language texts.
}

\bioauthor[:]{Yoshihide Kato}{%
Yoshihide Kato received the B.E. degree, the M.E. degree, and the Dr. of Engineering from Nagoya University in 1997, 1999, and 2003, respectively. 
He is currently an Associate Professor at Information \& Communications, Nagoya University. 
His research interests include natural language processing and computational linguistics.
}

\bioauthor[:]{Shigeki Matsubara}{%
Shigeki Matsubara received the B.E. degree from Nagoya Institute of Technology in 1993, and the M.E. degree, and the Dr. of Engineering from Nagoya University in 1995 and 1998, respectively. 
After becoming an Assistant Professor and an Associate Professor at Nagoya University, he became a Full Professor in 2017. 
His research interests include natural language processing and digital library. 
}
\end{biography}

\biodate   %%%% Following settings will be edited by editors.


\end{document}





