\documentclass[english]{jnlp_3.6}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{hangcaption_jnlp}
\usepackage{array}
%%%%\usepackage{udline}
%%%%\setulminsep{1.2ex}{0.2ex}
%%%%\let\underline=\ul
\allowdisplaybreaks


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
\usepackage{bm}
%%%%\usepackage{color}
%%%%\usepackage{fancybox}
%%%%\usepackage{ascmac}
%%%%\renewcommand\UrlFont{\rmfamily}
%%%%\usepackage{ulem}
%%%%\usepackage{xcolor}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{30}
\Number{2}
\Month{June}
\Year{2023}
%%%%\Issuetitle{「　　　　」特集号}

\received{2022}{10}{28}
\revised{2023}{2}{8}
\accepted{2023}{3}{6}

\setcounter{page}{689}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Exploring how well Large-scale Masked Language Models can Recognize Grammatical Errors}
\eauthor{Manabu Kimura\affiref{Author_1} \and Ryo Nagata\affiref{Author_2} \and Kazuaki Hanawa\affiref{Author_3}}
\eabstract{%
In this paper, we explore the capacity of a language model-based method for grammatical error detection in detail. We first show that 5 to 10\% of training data are enough for a BERT-based error detection method to achieve performance equivalent to what a non-language model-based method can achieve with the full training data; recall improves much faster with respect to training data size in the BERT-based method than in the non-language model method. This suggests that (i) the BERT-based method should have a good knowledge of the grammar required to recognize certain types of error and that (ii) it can use the knowledge to estimate whether the given word is correct or erroneous after fine-tuning with few training samples, which explains its high generalization ability in grammatical error detection. We further show with pseudo error data that it actually exhibits such nice properties for recognizing various types of error. Finally, based on these findings, we discuss a cost-effective method for detecting grammatical errors with feedback comments to learners.}

\ekeywords{Pre-trained Language Models, Masked Language Models, Grammatical Error Detection, Feedback Comment Generation, Generalization Ability }

\headauthor{Kimura et al.}
\headtitle{Exploring how well Masked Language Models can Recognize Grammatical Errors}

\affilabel{Author_1}{}{GRAS Group, Inc.}
\affilabel{Author_2}{}{Konan University}
\affilabel{Author_3}{}{RIKEN}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

\renewcommand{\thefootnote}{}
\footnote[0]{This paper is an extended version of that published in the Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics \cite{nagata-etal-2022-exploring}.}
\renewcommand{\thefootnote}{\arabic{footnote}}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}\label{sec:intro}

Although recent studies have empirically proven the effectiveness of language models pre-trained on a large corpus (hereafter, simply language models) in grammatical error detection and correction, there is still plenty of room for investigation to fully understand their capacity to detect and correct grammatical errors. 
As we will see in Sect.\,\ref{sec:related_work}, studies \cite{cheng-duan2020,kaneko_komachi2019,kaneko2020,didenko_shaptala2019} have shown that methods based on language models outperform other methods in grammatical error detection/correction.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1
\begin{figure}[b]
\begin{center}
\includegraphics{30-2ia15f1.pdf}
\end{center}
\caption{Detection Performance in Relation with Training Size (NUCLE).}
\label{fig:performance_on_nucle}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

In this paper, we reveal new aspects concerning the capacity of language models to detect grammatical errors to expand the previous studies.
First, using seven learner corpora, we show that an error detection method based on a BERT \cite{devlin-etal-2019-bert} model incredibly quickly learns to recognize various types of errors, as illustrated in Figure \ref{fig:performance_on_nucle}.
The key observation here is that it exhibits a $\log$-like performance curve that saturates at an early stage, outperforming the non-BERT model-based method (a BiLSTM model-based) with the full training data.
We observe the same tendency with a wide variety of writing topics, writers' proficiencies, and even out-domain settings where the domains of the training and test data are different.
These observations suggest that the BERT model-based method (hereinafter referred to as the BERT-based method)
(i) should have a good knowledge of the grammar required to recognize certain types of error by pre-training 
and (ii) 
should be able to use the knowledge to estimate whether the given word is correct or erroneous after fine-tuning with a few training instances. 
Following this, we further explore with real and pseudo error data why and where it gains in error detection, revealing the insights of language model-based methods. 
Specifically, we reveal that the contrast between the performance curves can be seen for all ten error types we examine except for spelling errors.
Even more specifically, we reveal that the BERT-based method, trained on a few training instances (as few as two!), can detect unseen errors.
For instance, given instances of a transitive verb with a preposition (e.g., \textit{*discuss about}), it recognizes unseen, erroneous combinations falling into the same error type (e.g., \textit{*approach to} and \textit{*attend in}).
Finally, based on the results, we describe the reason for the high performance of the BERT-based method even with a small amount of training, and imply that it should generally apply to grammatical error detection methods based on language models.%

These findings shed light on an important aspect of methods based on language models in practice.
It would be cost-effective to create a small amount of training instances (e.g., 500) to train a model; the gain would be much smaller after this point.
This decreases the cost required for grammatical error detection in two ways: by lowering the cost to create training data and the computation cost required to train a model.
We also discuss a cost-effective method for detecting grammatical errors with fine-grained types and detailed feedback comments \cite{hanawa-etal-2021-exploring}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Work}\label{sec:related_work}

\citeA{rei2017} shows it is useful for neural error detection models to introduce a secondary language model objective together with the main error detection objective. \citeA{rei_yannakoudakis2017} compare several other auxiliary training objectives including Part-Of-Speech (POS) tagging and error type identification and find that the language model objective is the most effective. This line of work suggests that grammatical error detection benefits from language modeling, although these studies use BiLSTM-based language models instead of masked language models pre-trained on a large corpus.


As mentioned in Sect.\,\ref{sec:intro}, several researchers have applied masked language models including BERT models to grammatical error detection and correction. \citeA{cheng-duan2020} and \citeA{kaneko_komachi2019} show that error detection methods gain in recall and precision with the use of language models. \citeA{bell2019} use BERT-based contextual embeddings for grammatical error detection and compare it with other types of contextual embedding. They show the BERT-based contextual embeddings are effective in almost all error types provided by ERRANT \cite{bryant}, although a BERT model is only used to obtain contextual embeddings, meaning that it is not fine-tuned, in their study. \citeA{yuan-etal-2021-multi} compare BERT, XLNet \cite{NEURIPS2019_dc6a7e65}, ELECTRA \cite{Clark2020ELECTRA} in grammatical error detection to show their effectiveness in grammatical error detection.\footnote{ELECTRA is not a language model. It however contains an architecture similar to that of a language model.} \citeA{kaneko2020} and \citeA{didenko_shaptala2019} also show performance improvements in grammatical error correction. To strengthen the findings of these previous studies, we will explore why and where error detection methods benefit from language models, revealing their generalization ability, in the following section.

There are studies available on the linguistic knowledge acquired by pre-trained BERT models.
\citeA{jawahar-etal-2019-bert} show that phrase-level information is acquired in the lower layers and syntactic information in the intermediate layers of the model.
The study also investigates the prediction performance of the subject-verb agreement relation by controlling the number of words between the subject and the verb.
 \citeA{wei-etal-2021-frequency} report thoughtful results by focusing on the subject-verb agreement task with a scratch pre-training. 
They report that subject-verb combinations that do not appear in the pre-training data can be predicted with high generalization performance and that the subject-verb combinations that do appear in the pre-training data are affected by absolute and relative frequencies, based on predictions made by a masked word [MASK]. 
In addition,  \citeA{NEURIPS2019_159c1ffe} and  \citeA{chi-etal-2020-finding} show that pre-trained BERT models and multilingual BERT models have the ability to discriminate dependency structure relationships (``nsubj'', ``obj'', etc.) between two words in a sentence. 
Although these studies \cite{jawahar-etal-2019-bert,NEURIPS2019_159c1ffe,chi-etal-2020-finding,wei-etal-2021-frequency} did not investigate the effects of fine-tuning, they provide clues as to ``why the BERT-based method achieves good performance with less training data''.

There has been a long history of studies that investigate the linguistic knowledge of language models including the work by \citeA{li2021,ettinger2020,warstadt2020} to name a few. 
A popular approach is to test whether a language model assigns a higher likelihood to an appropriate word than an inappropriate one, given a specific context.
The linguistic knowledge to be explored ranges from syntactic/semantic knowledge to common sense. These studies mostly use 
(i) synthetic test data: sentences that are generated synthetically by using a certain kind of template 
or (ii) perturbed test data: sentences that are generated by perturbing a natural corpus. 
Our work is different from these previous studies in two ways:
(i) to our best knowledge, we examine linguistic phenomena that have not been explored before in the previous studies (e.g., subjects marked with a preposition and errors involving the usages of transitive and intransitive verbs);  
(ii) we use real learner corpora with real errors as our test data.%


\citeA{mita2021} examine if an encoder-decoder neural network for grammatical error correction (not BERT-based) can learn the knowledge of grammar through the task of grammatical error correction. 
They target five error types: subject-verb agreement, verb form, word order, adjective/adverb comparison, and noun number. 
They use both synthetic and real learner data. 
They report a negative answer to the research question except for word order errors. 
They also report that their model learns the knowledge to detect the target errors in their synthetic data. 
However, there is still room for debate in this argument because error positions tend to be rather obvious in their synthetic data (e.g., adjective forms erroneously used as adverbs almost always appear at the end of a sentence). 
Our study expands and deepens their findings for a wider variety of error types that are much more difficult to detect (in that it requires a much wider range of linguistic knowledge including POS, lexical, and syntactic knowledge).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Data}\label{sec:data}

In this paper, we use two kinds of data: real and pseudo error data. Real data are error-annotated learner corpora while pseudo error data are automatically generated by perturbing a native English corpus.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[b]
\input{15table01.tex}%
\caption{Statistics on Real Datasets.}\label{tab:stats}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Real Data}\label{subsec:real_data}

For the real data, we use seven English learner corpora: the NUS Corpus of Learner English (NUCLE) \cite{dahlmeier-etal-2013-building}; the First Certificate of English error detection dataset (FCE) \cite{yannakoudakis2011}; the A to C sub-corpora in the BEA-2019 shared task dataset (BEA) \cite{bryant-etal-2019-bea};\footnote{We count the A, B, and C sub-corpora in BEA as three corpora because their writers are different and have different proficiency levels (A, B, and C in CEFR).}
the International Corpus Network of Asian Learners of English (ICNALE) \cite{ishikawa} annotated with feedback comments \cite{nagata18}.\footnote{Unlike BEA, we regard the two sub-corpora in ICNALE as a single corpus because its writers are identical.}
We use the data splits provided by the creator for FCE. 
BEA actually provides only training and validation data, and we use the original validation data as test data in this paper, a part of the original training data as validation data---concretely the same number as the original validation data--- and the rest as training data.
For all other corpora data partitioning into training, validation, and test data are not provided.
For ICNALE we randomly split the essays into training, validation, and test sets in the ratios of 85\%, 7.5\%, and 7.5\%, respectively; NUCLE uses 80\%, 10\%, and 10\% splits. 
Table~\ref{tab:stats} shows their statistics.

We use ICNALE to investigate in detail why and where language model-based methods gain an advantage. 
Their essay topics are controlled; they are written on 
either (a) \textit{it is important for college students to have a part-time job} 
or (b) \textit{smoking should be completely banned at all the restaurants in the country}, 
which hereafter will be referred to as \textit{PART-TIME JOB} and \textit{SMOKING}, respectively. 
Each essay is manually annotated with errors in preposition use and their corresponding feedback comments. Accordingly, the target errors are those marked as preposition errors in this corpus. 
For example, the major errors in the corpus include deverbal prepositions (e.g., \textit{*include} $\to$ \textit{including}), intransitive verbs with a direct object (e.g., \textit{*agree it} $\to$ \textit{agree with it}), 
a verb phrase used as a noun phrase (\textit{*Learn English is difficult.} $\to$ \textit{To learn/Learning English is difficult.}), 
and comparison between a phrase and a clause (e.g., \textit{*because an error} $\to$ \textit{because of an error}); see the work \cite{nagata18} for the details. 
On the other hand, NUCLE, FCE and BEA are tagged with various errors, including preposition errors, article errors, etc., although they are not specialized for preposition errors like ICNALE.

To investigate the relationship between the number of training sentences and detection performance, we randomly sample 100, 300, 500, 1,000, 3,000, 5,000, and 10,000 (9,000 instead of 10,000 for BEA)\footnote{In A and C in BEA, we sample 9000 sentences because they consist of less than 10000 sentences; although B consists of more than 10000 sentences, we also sample 9000 sentences for B to match the sample numbers for A and C.} 
sentences, 
resulting in eight sets of training data for each corpus. 
Note that these training sets (also the validation and test sets) contain error-free sentences.%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Pseudo Error Data}\label{subsec:pseudo_error_data}

For the pseudo error data, we use the 1998-2000 New York Times in the AQUAINT Corpus of English News Text \cite{aquaint} as a base corpus. 
We automatically generate erroneous sentences by injecting errors into them (one error per sentence). 
We first obtain chunks and parses by using spaCy.\footnote{\url{https://spacy.io/}} 
Here, we only use sentences longer than three tokens and shorter than 26 tokens to obtain reliable chunks and parses.


While we target all errors labeled as errors in the real data, we only target the following five error types in the pseudo error data:%
\begin{description}
\setlength{\parskip}{-0.7mm}
\item[Prepositional infinitive:] \textit{to}-infinitive with prepositions other than \textit{to}\\ 
(e.g., \textit{*a book \underline{for} read})
\item[Subject verb:] Verb phrases used as a subject\\
(e.g., \textit{*\underline{Learn English} is difficult.})
\item[Preposition + subject:] Subjects used with a preposition\\
(e.g., \textit{*\underline{In} the restaurant serves good food.})
\item[Transitive verb + preposition:] Transitive verbs used with a preposition\\
(e.g., \textit{*We discussed \underline{about} it.})
\item[Intransitive verb + object:] Intransitive verbs taking a direct object\\
(e.g., \textit{*We \underline{agree it}.})
\end{description}
These five error types are selected for the following reasons:
(i) they are the major errors in ICNALE; 
(ii) we can write a software program to generate pseudo errors based on chunks and parses.

We inject an error into a sentence by adding, replacing, or removing a preposition based on the analysis obtained by spaCy. 
When adding a preposition, we randomly choose one of the following five prepositions: \textit{at}, \textit{about}, \textit{to}, \textit{in}, and \textit{with}.
This addition includes \textbf{preposition + subject} and \textbf{transitive verb + preposition}. 
We can find the subject of a sentence from its parse and add a randomly-chosen preposition before the subject noun phrase
(e.g., \textit{The restaurant serves good food.} $\to$ \textit{*In the restaurant serves good food.}). 
We can do the same for \textbf{transitive verb + preposition} by adding a randomly-chosen preposition before the direct object (noun phrase) of the transitive verb. We only apply replacement to \textbf{preposition infinitive} where we replace \textit{to} in \textit{to}-infinitive with the preposition \textit{for}. 
Again, we can find \textit{to}-infinitives using the POS-tagging and parsing results. 
For \textbf{subject verb}, we first find a \textit{to}-infinitive used as the subject of a sentence and then remove the \textit{to}. For \textbf{intransitive verb + object}, 
we remove the token if it meets the following conditions: 
(i) appearing on the right of the intransitive verb, 
(ii) recognized as a preposition by spaCy, 
(iii) attached to the intransitive verb, and 
(iv) nearest to the intransitive verb.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 2
\begin{table}[b]
\input{15table02.tex}%
\caption{Target Transitive and Intransitive Verbs.}\label{tab:transitive_verbs}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Table~\ref{tab:transitive_verbs} shows the target transitive and intransitive verbs. 
It should be emphasized that, as shown in the table, there is no overlap of verbs in the training/validation and test data.
This means that error detection methods must learn to detect these two types of errors in a set of verbs from those in another set of verbs.%

From the resulting pseudo error data, we randomly sample $2^i (1 \leq i \leq 10)$ sentences for each error type, resulting in ten sets of training data 
(e.g., when $i=1$, the set comprises two instances of each error type, yielding ten instances ($2^1\times5$ error types) in total). 
We use these training sets to estimate the relationship between the number of training sentences and detection performance. 
For a validation set, we randomly sample 200 sentences for each error type. 
Similarly, we use a test set consisting of 200 sentences randomly sampled for each error type along with 200 error-free sentences.
The validation and test sets are fixed regardless of the training data. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Grammatical Error Detection Methods}\label{sec:methods}

This section describes the BERT-based method and three other methods for comparison. 
Before looking into them, let us define grammatical error detection formally. 
Grammatical error detection can be solved as a token classification problem.\footnote{Generally, it can also be solved as a sequence labeling problem using, for example, CRF.However, \citeA{rei2017} shows that the grammatical error detection task does not benefit from CRF. We observed the same tendency in our datasets, and most errors consist of one token. Accordingly, we solve it as a token classification problem because error labels spanning multiple tokens can be treated as a single token by considering only the central token.} 
We will denote a sequence of words and its length by $w_{1}, \dots, w_{i} \dots, w_{N}$ and $N$, respectively. We will denote the corresponding sequence of labels by $l_{1}, \ldots, l_{i}, \ldots, l_{N}$, where $l_i$ corresponds to the label of $w_{i}$. We assume two sets of labels: 
(i) either C or E denoting \textit{correct} or \textit{erroneous}, respectively, in the real data,  
and (ii) $K$ labels for $K$ error types and C for \textit{correct} in the pseudo error data. 
Then, grammatical error detection is defined as the problem of predicting the optimal label sequence given $w_{1}, \dots, w_{i} \dots, w_{N}$. 


We use neural networks to predict the optimal label sequence. 
In this paper, training is repeated five times with different (but fixed) random seeds.
The reported performance values (i.e., recall, precision, and $F_{1.0}$) are averaged over the five runs. 
Training epochs are 50 for the real data and 10 for the pseudo error data at the maximum, and we adopt the epoch achieving the best $F_{1.0}$ on the validation set. 
Other major hyper parameters are shown in Appendix~\ref{app:hyper_params}.


Under these specifications, the BERT-based method is formalized as follows. 
It takes a word sequence $w_{1}, \dots, w_{i} \dots, w_{N}$ as input and gives the corresponding optimal label sequence $l_{1}, \ldots, l_{i}, \ldots, l_{N}$ as output by conducting the following procedures:%

\begin{description}
\item[\textbf{(1) Subword:}] convert all $w_{i}$ into their corresponding subwords as follows:  
\begin{equation}
s_{i,0},\cdots,s_{i,j},\cdots,s_{i,\#(i)}
\end{equation}
where $\#(i)$ denotes the number of subwords corresponding to $w_{i}$. 

\item[\textbf{(2) Encode:}] encode all $s_{i,j}$ into BERT embeddings $\bm h_{i,j}$ using the following equation:
\begin{equation}
\bm{h}_{i,j} = \mbox{BERT}(s_{i,j})
\end{equation}
where $\mbox{BERT}(\cdot)$ denotes a BERT model taking subwords as input and outputs their corresponding embedding vectors of $h$-dimension (specifically, $h=768$ for `bert-base') from the final layer. We use `bert-base-uncased' for $\mbox{BERT}(\cdot)$. 

\item[\textbf{(3) Token classification:}] output the optimal labels using the following equation:
\begin{equation}
l_{i,j} = \arg\max\mbox{softmax} (\bm{Wh}_{i,j})
\end{equation}
where $\bm W$ is a $k\times h$ weight matrix and $k$ is either $2$ or $K+1$ (the number of different labels). 
We accept $l_{i,0}$ as output, with $l_{i}$ corresponding to $w_{i}$.
\end{description}

For comparison, we use a BiLSTM-based error detection method.
It follows the above-mentioned steps (1) to (3) but uses a BiLSTM model as an encoder instead of a BERT model.
The input word sequence is turned into a sequence of embedding vectors, where each embedding vector consists of the concatenation of the conventional word embedding and a character-based embedding. 
The character-based embedding is obtained by another BiLSTM taking the characters of each word following the approach described in the study \cite{akbik2018}.  
The concatenated embeddings are put into the encoder BiLSTM to produce vectors for classification in step (3). 
Specifically, we use the implementation FLAIR \cite{akbik2019}. 
We will refer to this method as the BiLSTM-based method hereafter.


We also investigate the effectiveness of fine-tuning and pre-training BERT models.
For fine-tuning, the BERT model part of the BERT-based method is fixed during training, and only the output layer is adjusted by the training data;
for pre-training, all weights of Transformer blocks and the pooler layer within the BERT model part are randomly initialized by $\mathcal{N} (0, 0.02^{2})$, 
as in the study \cite{devlin-etal-2019-bert}.
Hereafter, we will refer to the former and the latter as the BERT-based method without fine-tuning and pre-training, respectively.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Evaluation}\label{sec:evaluation}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Overall Performance on Real Data}\label{subsec:overall_performance}

The figure on the left in Figure \ref{fig:performance_on_nucle} (on the second page) shows the relationship between the number of training sentences and $F_{1.0}$ in NUCLE. 
The figure reveals the high generalization ability of the BERT-based method. 
Notably, it exhibits a log-like performance curve with a saturation point at an early stage (500--1,000 training sentences). 
In contrast, the BiLSTM-based method improves linearly with the number of training sentences but achieves a much lower $F_{1.0}$.
Surprisingly, the BERT-based method with only 100 training sentences outperforms the BiLSTM-based method with the full training set, which is approximately 170 times larger. 


To further investigate the results, we consider precision-recall curves
(the right figure; the numbers at the edges of each curve correspond to the numbers of minimum and maximum training sentences, and the plots are lined up in minimum to maximum order). 
The precision-recall curves show that only the BERT-based method quickly improves in recall with an increasing number of training sentences, while both methods improve in precision. 
In other words, only the BERT-based method learns to recognize various error types with little exposure to error examples. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 2
\begin{figure}[t]
\begin{center}
\includegraphics{30-2ia15f2.pdf}
\end{center}
\caption{Detection Performance in Relation with Training Size (FCE).}
\label{fig:performance_on_fce}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


Figure \ref{fig:performance_on_fce} and Figure \ref{fig:performance_on_bea} with FCE and BEA, respectively, also show similar tendencies. 
It is important to note that the writing topics are diverse in the corpora and that the $F_{1.0}$ curves for all proficiency levels (A, B, and C)\footnote{The proficiency levels A, B, and C in BEA, which correspond to those in CEFR, become higher in this order.} in BEA (Figure \ref{fig:performance_on_bea}) exhibit the same trend as the other corpora, although the performance values differ depending on the levels.
These results empirically confirm that the high generalization ability of the BERT-based method holds in various writer populations and writing topics.%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Effects of Fine-tuning and Pre-training}\label{subsec:performance_fine-tuning_pre-training}

We now move on to the investigation into the effects of fine-tuning and pre-training. 
Figure \ref{fig:performance_on_fce_rand_init} shows the relationship between the number of training sentences and performance with/without fine-tuning and pre-training in FCE.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 3
\begin{figure}[t]
\begin{center}
\includegraphics{30-2ia15f3.pdf}
\end{center}
\hangcaption{Detection Performance in Relation with Training Size (BEA). A, B, and C correspond to the CEFR levels. The experimental conditions are referred to as the labels such as ``BERT:A'' and ``BERT:B'' in the legend.}
\label{fig:performance_on_bea}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 4
\begin{figure}[t]
\begin{center}
\includegraphics{30-2ia15f4.pdf}
\end{center}
\hangcaption{Comparison between BERT-based Methods with/without Fine-tuning and Pre-training (FCE).}
\label{fig:performance_on_fce_rand_init}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

As expected, fine-tuning and pre-training prove crucial for the BERT-based method.
The $F_{1.0}$ values almost double at all points by fine-tuning and pre-training.
Hence, this supports our hypotheses that a pre-trained BERT model 
(i) should have a good knowledge of the grammar acquired by pre-training to recognize certain types of error and (ii) 
should be able to use the knowledge to estimate whether the given word is correct or erroneous after fine-tuning with a few training instances. 
A possible reason why fine-tuning (of the BERT model part) works well is that it requires much more degrees of freedom in terms of the network parameters 
to fit the linguistic knowledge acquired during pre-training for detecting a wide variety of grammatical errors, which have a certain degree of complexity.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Cross-corpora evaluation}\label{subsec:cross-corpora_evaluation}

One might argue that the BERT-based method learns so quickly because it is trained on a learner corpus in one domain and is tested on the same domain. Conversely, it might not perform well if the domains of training and test data are different.


To investigate this point, in this subsection, we report on results of cross-corpora evaluation. 
Specifically, we train the BERT-based method on FCE, NUCLE, and A to C of BEA and test them on FCE. 
In this setting, the writers, their proficiency levels, and writing topics are all different among the corpora (of course, except for FCE).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 5
\begin{figure}[b]
\begin{center}
\includegraphics{30-2ia15f5.pdf}
\end{center}
\hangcaption{Cross-corpora evaluation (Train the BERT-based methods on FCE, BEA, and NUCLE and test them on FCE). The bold line means that the training data is the same corpus, FCE, as the test data.}
\label{fig:cross_corpora_evaluation}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Figure \ref{fig:cross_corpora_evaluation} shows the results of cross-corpora evaluation. It gives the negative answer to the above argument; the BERT-based method learns quickly regardless of in-domain or out-domain settings (in particular, Train:BEA-A and Train:BEA-B).
All performance curves exhibit the now-familiar characteristics: the log-like curve and the performance saturation at 500--1,000 training sentences.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Performance with Detailed Error Types}\label{subsec:detailed_performance}

For better understanding, we now investigate detection performance with detailed error types. 
Figure \ref{fig:fce_error_typed} shows recalls calculated per detailed error type\footnote{PUNCT, OTHER, and those whose frequency is less than 150 are excluded in Figure \ref{fig:fce_error_typed}.} in FCE, where error types are automatically obtained by using ERRANT.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 6
\begin{figure}[b]
\begin{center}
\includegraphics{30-2ia15f6.pdf}
\end{center}
\caption{Detection Performance (Recall) per Error Type in Relation with Training Size (FCE).}
\label{fig:fce_error_typed}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

From Figure \ref{fig:fce_error_typed}, we observe that the BERT-based method learns to detect most of the spelling errors (SPELL) from the beginning while the BiLSTM-based method shows a much more gradual rise. 
This is because the BERT model is pre-trained on large native corpora and thus has a large vocabulary set. 


More importantly, it attains performance saturation at an early stage (in 500--1,000 training sentences!) for all errors, resulting in $\log$-like curves, whereas the BiLSTM-based method improves rather linearly (except for SPELL).
Typical examples are DET (determiner errors) and NOUN:NUM (errors in noun number). 
The notion of noun countability with POS plays an important role in detecting these two types of errors, as in \textit{*I am student/countable.} and \textit{*an evidence/uncountable}.
This suggests that a pre-trained BERT model has some knowledge about noun countability and POS (singular/plural nouns). 
Hence, we speculate that the model understands broad linguistic properties, which we will empirically prove with ICNALE and the pseudo error data.%




To this end, focusing on preposition errors, we investigate the performance of BERT-based and BiLSTM-based methods in the in-domain/out-domain training settings of ICNALE with manually-annotated errors in preposition use. 
Note that the other error types are unmarked and are treated as false positives if detected. 

The performance curves shown in Figure \ref{fig:performance_icnale} have the same characteristics observed so far. 
This means that the BERT-based method learns to recognize only specified error types  (in this case, errors in preposition use) by fine-tuning with a small amount of training data. 
This holds regardless of the in-domain or out-domain training setting, as observed in Subsect.\,\ref{subsec:cross-corpora_evaluation}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 7
\begin{figure}[t]
\begin{center}
\includegraphics{30-2ia15f7.pdf}
\end{center}
\hangcaption{Detection Performance in Relation with Training Size (ICNALE). The bold line means that the training data has the same topic as the test data.}
\label{fig:performance_icnale}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

A closer look at the detection results reveals that many errors require linguistic knowledge, including POS, syntactic relations, and lexical properties such as transitive/intransitive verbs. 
For example, ``Preposition + subject'' errors, which were introduced in Subsect.\,\ref{subsec:pseudo_error_data}, require the notions of POS such as verbs and syntactic relations such as subjects. 
Similarly, the distinction between transitive and intransitive verbs plays a crucial role in ``transitive verb + preposition'' and ``intransitive verb + object''. % 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Performance on Pseudo Error Data}\label{subsec:performance_on_synthetic_data}

To strengthen the analysis, we now turn to the detection performance of the BERT-based method on the pseudo error data with six classification categories (five error types plus correct). 
As described in Subsect.\,\ref{subsec:pseudo_error_data}, we train it on the ten sets of training data (i.e., 2, 4, $\cdots$, 1024 training sentences for each error type) and test the trained models on the fixed test set. 
We then estimate the relationship between the number of training sentences and detection performance for each error type.%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 8
\begin{figure}[b]
\vspace{-1\Cvs}
\begin{center}
\includegraphics{30-2ia15f8.pdf}
\end{center}
\caption{Detection Performance per Error Type in Relation with Training Size (Pseudo Error Data).}
\label{fig:nyt_by_nyt_f1}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Figure \ref{fig:nyt_by_nyt_f1} shows the relationship between the size of training data and $F_{1.0}$ for each error type. 
It is seen that the BERT-based method recognizes some of the target errors at early stages (even with two or four training sentences). 
Performance increases drastically even with eight training sentences for most of the error types except ``Intransitive verb + object'' error type. 
For instance, the BERT-based method recognizes more than half of the ``Preposition + subject'' errors with a precision of 0.800 after only eight training instances. 
Without knowledge about the notions of POS such as verbs and syntactic relations such as subjects, 
it would be difficult to achieve a similar performance in this type of error considering that the noun phrase of a subject and its position in the sentence considerably vary depending on the target sentence.



The same argument applies to ``transitive verb + preposition'' and ``intransitive verb + object''. 
It should be emphasized that the BERT-based method must detect errors in verbs that never\footnote{Some of the verbs may appear in the training sentences for the other error types. However, they never appear in the erroneous phrases. Also, they do not appear when the training size is small.} appear in the training data; recall that there is no overlap of the target transitive/intransitive verbs in the training and test data. In other words, the BERT-based method can recognize unseen, erroneous combinations. 
The actual instances in the corpus include \textit{*visited in Atlanta} (transitive verb + preposition error type) and \textit{*specialized environmental litigation} (intransitive verb + direct object error type), after just seeing \textit{*mention in}, \textit{*discussed about}, \textit{*were related drugs}, and \textit{*belongs Lon's grandmother}. 
These training and test sentences have nothing much in common except that they are combinations of transitive/intransitive verbs and prepositions/objects. 
Besides, the appearance of correct combinations of other verbs and prepositions/objects in the test data makes the task even more difficult without knowledge of POS and syntactic relations. 
All the findings described in Subsect.\,\ref{subsec:overall_performance} to~\ref{subsec:performance_on_synthetic_data} support our hypotheses that a pre-trained BERT model has linguistic knowledge and that 
it can use this knowledge to estimate whether the given word is correct or erroneous after fine-tuning.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Discussion}\label{sec:implication}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Interpreting the Capacity}\label{subsec:explanation}

So far, we have observed the generalization ability of the BERT-based method in grammatical error detection. 
At first sight, it would appear almost impossible to recognize a particular type of error after seeing just a few training examples, which is actually not as shown below.

It would be so if the method did not know about canonical English. 
However, since the BERT model is a language model, it can accurately estimate how probable a given sequence is.\footnote{Strictly, the BERT model can only estimate the probabilities of words appearing in a given sequence.}
Conversely, it can tell which words are improbable in the given sequence, which are potentially grammatical errors. 
Flagging words with low probabilities would achieve a good recall. 
Such words may not necessarily be grammatical errors, and this way of error detection would cause a large number of false positives. 
Fine-tuning plays an important role here as it informs the BERT model (or the BERT-based method) about which sequences are erroneous.
It should be emphasized that, words, phrases, and syntax are abstractly represented in neural language models, including the BERT model \cite{jawahar-etal-2019-bert,warstadt2020,NEURIPS2019_159c1ffe,chi-etal-2020-finding}.
This helps the BERT-based method to quickly learn the erroneous sequences by fine-tuning. 
In fact, even for the BERT model without fine-tuning, its abstract representation has been shown to have generalization ability in the subject-verb agreement task \cite{wei-etal-2021-frequency}. 
As demonstrated in Subsect.\,\ref{subsec:performance_on_synthetic_data} (and also in the previous studies  \cite{li2021,ettinger2020,warstadt2020}), from pre-training, the BERT model recognizes certain groups of words such as prepositions, verbs, and the group of verbs that do not take a preposition right after. 
Additionally, it knows the relationships between these groups, which are the dependency structure relationships such as ``det'', ``obj'', and et al. \cite{NEURIPS2019_159c1ffe,chi-etal-2020-finding}. 
With this kind of knowledge, it is not difficult to detect unseen erroneous sequences if they fall into one of the known error types. 
This should be the key ability of the BERT-based method to perform well, as we have observed in the five detailed error types in Subsect.\,\ref{subsec:detailed_performance} and Subsect.\,\ref{subsec:performance_on_synthetic_data}.



This argument will perhaps be applicable to error detection methods based on language models other than the BERT model as long as they estimate the probability of a given word in a sentence accurately with the abstract representation of words, phrases, and broader constructions.
It does not have to be a language model. 
We only require a function that estimates the probability of a certain event in each sequence.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 9
\begin{figure}[b]
\begin{center}
\includegraphics{30-2ia15f9.pdf}
\end{center}
\caption{Comparison between ELECTRA and BERT-based Methods (FCE).}
\label{fig:performance_on_fce_electra}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

To support this claim, we show the performance of an error detection method based on an ELECTRA model, which is pre-trained as a discriminator with essentially the same architecture as the BERT model. 
An ELECTRA model estimates the probability of whether the given words are real or automatically generated in the given sequence. 
Here, we replace the BERT model with an ELECTRA model in the BERT-based method. 
Figure \ref{fig:performance_on_fce_electra} shows the performance of the ELECTRA-based method\footnote{We use `electra-base-discriminator' as the ELECTRA model.} on FCE along with the BERT-based method. 
As expected, the ELECTRA-based method exhibits a similar performance curve to that of the BERT-based method. 
In addition, the ELECTRA-based model outperforms the BERT-based model.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Cost-Effective Error Detection with Feedback Comments}\label{subsec:cost-effecitive_method}

The findings we have obtained so far suggest the possibility of a cost-effective system that accurately detects grammatical errors and recognizes the detailed error types.
For example, by creating a few instances of the erroneous combination of transitive verbs and prepositions manually or automatically, as we saw in the previous sections (e.g., \textit{*discuss about}), one can develop a system detecting the same types of errors in other transitive verbs and prepositions (e.g., \textit{*attend in it}).
With the detailed error types, the system can also provide feedback to the user instead of just indicating them as preposition errors, such as \textit{Transitive verbs do not take a preposition. Instead, they take a direct object}.




We trained the BERT-based method on the pseudo error data and tested it on the real (learner) data to examine the above possibility. 
We manually annotated the real data with five error types based on the feedback comments available in ICNALE. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 10
\begin{figure}[b]
\begin{center}
\includegraphics{30-2ia15f10.pdf}
\end{center}
\hangcaption{Detection Performance ($F_{1.0}$) on per Error Type in Relation with Training Size: Trained on Synthetic Data; Tested on Real Data (PART-TIME JOB and SMOKING merged).}
\label{fig:both_by_nyt_f1}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Figure \ref{fig:both_by_nyt_f1} shows the results. 
It reveals that the BERT-based method trained on the pseudo error data does not perform as well on the real data as it does on the pseudo error data.
Performance growth stops at an early stage (around eight training sentences).
This is likely because errors in the pseudo data cover only part of the errors in the real data. 
In addition, multiple errors appearing in a sentence make the task harder in the real data.
Besides, the error rate is much lower in the real data than in the pseudo error data, where one error occurs per sentence except for 200 error-free sentences (although multiple types of errors appear in the whole data set). 
These conditions make the task much more difficult with the real data, as \citeA{flachs} demonstrate.%



However, 
the results shown in Fig.\,\ref{fig:nyt_by_nyt_f1} still encourage us to develop language model-based systems with a small amount of in-domain training data in order to detect grammatical errors with detailed error types. 
One possible way to achieve this is to sample sentences from unannotated essays written on target topics and then annotate them with the specific error types for which the developer wants to give feedback to learners. 
This will naturally mitigate the problems caused by the occurrence of multiple errors of different types and the error rate difference. 
One can also manually create sample error sentences (and their correct versions). 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusions}\label{sec:conclusions}

In this paper, 
we have explored the capacity of a large-scale masked language model in recognizing grammatical errors. 
Our findings are summarized as follows: 
(1) a BERT-based error detection method has a high generalization ability, exhibiting a log-like performance with the first performance saturation at the point of around 500--1,000 training instances; 
(2) this holds for a wide variety of learner populations and writing topics;
(3) the high generalization ability suggests its potential in developing systems that detect and explain grammatical errors with very few training instances. 

Although the experiments were performed only with the standard-sized normal BERT model, bert-based-uncased, future work will explore the properties of other models and their differences as well. 
First, we plan to investigate with large-size BERT models to study the effect of parameter size.
Second, we plan to experiment with multilingual BERT models and other category-specific BERT models to study domain effects during pre-training.
Third, we plan to experiment with other language models, such as GPT-* models \cite{radford2019language} and XLNet models  \cite{NEURIPS2019_dc6a7e65}, to study differences between language models.

Moreover, 
we will explore effective methods for implementing cost-effective error detection systems with feedback comments. We will also expand this investigation to grammatical error correction.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment

This work was partly supported by the Japan Science and Technology Agency (JST), PRESTO Grant Number JPMJPR1758, Japan. 
This work was partly supported by JSPS KAKENHI Grant Number JP22K12326
and Japan Science and Technology Agency (JST), PRESTO Grant Number
JPMJPR1758, Japan. 
This work was partly conducted by using the computational resources of AI Bridging Cloud Infrastructure (ABCI) provided by the National Institute of Advanced Industrial Science and Technology (AIST).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.7}
\bibliography{15refs}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\appendix

\section{Hyper parameter settings}\label{app:hyper_params}
Table~\ref{tab:hyper_params} shows the major hyper parameters used in the experiments. Note that when we use the pseudo error data for training, the number of training sentences can be as small as ten, and we use a rather small batch of five; otherwise, we use 32.%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Libraries settings}
Table~\ref{tab:versions_of_libraries} shows the versions of the major libraries used in the experiments.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[h]
\input{15table03.tex}%
\caption{Major hyper parameters.}\label{tab:hyper_params}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 4
\begin{table}[h]
\input{15table04.tex}%
\caption{Libraries settings.}\label{tab:versions_of_libraries}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Manabu Kimura}{%
Manabu Kimura received his B.S. degree from the Kyoto Institute of Technology and his M.E. and Dr. Eng. from the Nara Institute of Science and Technology. He currently works in the Research \& Development group of GRAS Group, Inc. His current research interests include natural language processing and machine learning. He is a member of ACL.
}


\bioauthor[:]{Ryo Nagata}{Ryo Nagata graduated from the Department of Electrical Engineering, Meiji University in 1999, completed the doctoral program in information engineering at Mie University in 2005 and became a research associate at Hyogo University of Teacher Education. Since 2008, he has been an associate professor at Konan University. His research interests are language modeling for linguistics, grammatical error detection and correction, and feedback comment generation. He is a member of IEICE and ACL.
}

\bioauthor[:]{Kazuaki Hanawa}{%
Kazuaki Hanawa received the M.S. and Ph.D. degrees in system information sciences from Tohoku University, Sendai, Japan, in 2019 and 2022, respectively. In 2019, he joined the Center for Advanced Intelligence Project, RIKEN. His research interests include machine learning and natural language processing.
}
\end{biography}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\biodate



\end{document}

