\documentclass[english]{jnlp_3.7}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{hangcaption_jnlp}
\usepackage{array}
%%%%\usepackage{udline}
%%%%\setulminsep{1.2ex}{0.2ex}
%%%%\let\underline=\ul
\allowdisplaybreaks


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
\usepackage{bm}
%%%%\usepackage{graphics}
%%%%\usepackage{caption}
%%%%\usepackage{subcaption}
\usepackage{multirow}
%%%%\usepackage{adjustbox}
%%%%\usepackage{pgfplots}
%%%%\usepgfplotslibrary{dateplot,groupplots}
%%%%\usepackage{tikz}
\usepackage{diagbox}
%%%%\usepackage[multiple]{footmisc}
\usepackage{bbm}
\usepackage{booktabs}
%%%%\usetikzlibrary{patterns}
%%%%\usepackage{verbatim}
%%%%\usepackage{algorithm}  
%%%%\usepackage{algorithmicx}
%%%%\usepackage{algpseudocode}  
%%%%\usepackage{collcell}
    \usepackage[dvipdfmx]{xcolor}
%%%%\usepackage{CJKutf8}
%%%%\usepackage{arydshln}
%%%%\usepackage{pgfplotstable}
%%%%\DeclareMathOperator*{\argmax}{arg\,max}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{31}
\Number{1}
\Month{March}
\Year{2024}
%%%%\Issuetitle{「　　　　」特集号}
%%%%\Issuetitle{Special Issue: XXXXXXXX}

\received{2023}{5}{27}
\revised{2023}{8}{22}
\accepted{2023}{10}{12}

\setcounter{page}{3}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Bidirectional Transformer Reranker for Grammatical Error Correction}
\eauthor{Ying Zhang\affiref{Author_1} \and Hidetaka Kamigaito\affiref{Author_2} \and Manabu Okumura\affiref{Author_1}\affiref{Author_3}} 
\eabstract{%
Pre-trained sequence-to-sequence (seq2seq) models have achieved state-of-the-art results in the grammatical error correction tasks. However, these models are plagued by prediction bias owing to their unidirectional decoding. Thus, this study proposed a bidirectional transformer reranker (BTR) that re-estimates the probability of each candidate sentence generated by the pre-trained seq2seq model. The BTR preserves the seq2seq-style transformer architecture but utilizes a BERT-style self-attention mechanism in the decoder to compute the probability of each target token using masked language modeling to capture bidirectional representations from the target context. To guide the reranking process, the BTR adopted negative sampling in the objective function to minimize the unlikelihood. During inference, the BTR yielded the final results after comparing the reranked top-1 results with the original ones using an acceptance threshold $\lambda$. Experimental results showed that, when reranking candidates from a pre-trained seq2seq model, the T5-base, the BTR on top of T5-base yielded scores of 65.47 and 71.27 $F_{0.5}$ on the CoNLL-14 and building educational applications 2019 (BEA) test sets, respectively, and yielded 59.52 GLEU score on the JFLEG corpus, with improvements of 0.36, 0.76, and 0.48 points compared with the original T5-base. Furthermore, when reranking candidates from T5-large, the BTR on top of T5-base improved the original T5-large by 0.26 on the BEA test set.
}
\ekeywords{Grammatical Error Correction, Seq2seq, Reranking}

\headauthor{Zhang et al.}
\headtitle{Bidirectional Transformer Reranker for Grammatical Error Correction}

\affilabel{Author_1}{}{Tokyo Institute of Technology}
\affilabel{Author_2}{}{NARA Institute of Science and Technology (NAIST)}
\affilabel{Author_3}{}{RIKEN Center for Advanced Intelligence Project}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}

\vspace{-0.5\Cvs}
Grammatical error correction (GEC) is a sequence-to-sequence (seq2seq) task that requires a model to aim to correct an ungrammatical sentence. 
For instance, the ungrammatical sentence ``Disadvantage is parking their blue car is very difficult'' can be corrected to ``A disadvantage is that parking their cars is very difficult.''
Various neural models for GEC have been proposed \cite{Junczys-Dowmunt18,kiyono-etal-2019-empirical,kaneko-etal-2020-encoder,rothe-etal-2021-simple} owing to the importance of this task for language-learners who tend to produce ungrammatical sentences.\footnote{This paper is a revised version of Zhang et al. (2023), which has been accepted for publication at the Findings of the Association for Computational Linguistics: ACL 2023.}

Previous studies have shown that GEC can be approached as machine translation method by using a seq2seq model \cite{luong-etal-2015-effective} with a transformer \cite{NIPS2017_3f5ee243} architecture \cite{Junczys-Dowmunt18,zhao-etal-2019-improving,kiyono-etal-2019-empirical,kaneko-etal-2020-encoder,rothe-etal-2021-simple}.
As a neural model comprises an encoder and a decoder, the seq2seq architecture typically requires a large amount of training data.
Because GEC suffers from limited training data, applying a seq2seq model for GEC results in a low-resource setting. This can be handled by introducing synthetic data for training \cite{kiyono-etal-2019-empirical,omelianchuk-etal-2020-gector,stahlberg-kumar-2021-synthetic}. 
However, as pointed out by \citeA{rothe-etal-2021-simple}, the use of synthetic data in GEC may result in a distributional shift and require language-specific tuning, which can be time-consuming and resource-intensive.

Considering the limitations of the synthetic data, the current trend involves using the learned and general representations from a pre-trained model, such as BERT \cite{devlin-etal-2019-bert}, XLNet \cite{yang2019xlnet}, BART \cite{lewis2020bart}, and T5 \cite{2020t5}, which have been trained on large corpora and are effective for various downstream tasks.
According to \citeA{kaneko-etal-2020-encoder}, incorporating a pre-trained masked language model (MLM) into a seq2seq model could facilitate correction.
In addition, as reported by \citeA{rothe-etal-2021-simple}, the pre-trained T5 model achieved state-of-the-art results on GEC benchmarks for four languages after successive fine-tuning with the cleaned LANG-8 corpus (cLang-8) \cite{rothe-etal-2021-simple}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1
\begin{figure}[b]
\begin{center}
\includegraphics{31-1ia1f1.pdf}
\end{center}
\caption{Directed graphs for a sequence $\bm{x}=(x_{1}, x_{2}, x_{3})$, where $x_i$ is the $i$-th token in $\bm{x}$.}
\label{fig:directed_graphs}
%%%%\label{fig:bayes_net}
%%%%\label{fig:dependency_net}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Although the seq2seq model with pre-trained representations has shown to be effective for GEC, its performance is constrained by its unidirectional decoding\textcolor{black}{, wherein the prediction of the next token depends solely on the previously generated tokens \cite{hyndman2018forecasting}. This process can be represented by an acyclic directed graph (or Bayes net) \cite{murphy2012machine}, as shown in 
%%%%Figure \ref{fig:bayes_net} \cite{yamakoshi-etal-2022-probing}.}
    Figure \ref{fig:directed_graphs}a \cite{yamakoshi-etal-2022-probing}.}
As suggested by \citeA{liu-etal-2021-neural}, for an ungrammatical sentence, a fully pre-trained seq2seq GEC model \cite{kiyono-etal-2019-empirical} could generate several high-quality grammatical sentences using beam search. 
However, even among these candidates, a gap may exist between the selected hypothesis and the most grammatical one.
Our experimental results, presented in Table~\ref{table:main_result}, also demonstrate their investigation.
To solve this decoding problem, \textcolor{black}{bidirectional models, resembling cyclic directed graphs or dependency nets \cite{10.1162/153244301753344614} as shown 
%%%%in Figure~\ref{fig:dependency_net}, have been explored.}
    in Figure~\ref{fig:directed_graphs}b, have been explored.}
Given the hypotheses of a seq2seq GEC model, \citeA{kaneko-etal-2019-tmu} used \textcolor{black}{an encoder-only transformer model}, BERT, to classify between ungrammatical and grammatical hypotheses, and reranked them based on the classification results.
Previous studies by  \cite{kiyono-etal-2019-empirical,kaneko-etal-2020-encoder} also showed that the seq2seq GEC model decoding in an opposite direction, that is, right-to-left, is effective as a reranker for a left-to-right GEC model. 

Therefore, to further improve the performance of the pre-trained seq2seq model for GEC, it is essential to find ways to leverage the bidirectional representations of the target context.
In this study, based on the seq2seq-style transformer model, we proposed a bidirectional transformer reranker (BTR) to address the interaction between the source sentence and the bidirectional target context.
The BTR utilizes a BERT-style self-attention mechanism in the decoder to predict each target token by masked language modeling \cite{devlin-etal-2019-bert}.
Considering several candidate target sentences from a base model, the BTR can re-estimate the sentence probability for each candidate from the bidirectional representation of the candidate, which is different from the conventional seq2seq model.
\textcolor{black}{Distinct from encoder-only transformer models like BERT, the BTR employs the seq2seq architecture to address the GEC task. Moreover, in contrast to the approach of combining two separate opposite-directional models to incorporate bidirectional information by post-combining predicted probabilities, the BTR integrates bidirectional representation into the neural network to capture deeper representations and directly output probabilities.}
During training, for guiding the reranking, we adopted negative sampling \cite{NIPS2013_9aa42b31} for the objective function to minimize the unlikelihood while maximizing the likelihood. In inference, considering the robustness of pre-trained models, we compared the reranked top-1 results with the original ones considering an acceptance threshold $\lambda$ to decide whether to accept the suggestion from the BTR.

The state-of-the-art model for GEC \cite{rothe-etal-2021-simple}, a pre-trained transformer model, T5 (either T5-base or T5-large), was considered as our base model and its generated candidates were used for reranking. 
Because the BTR can inherit learned representations from a pre-trained transformer model, we constructed the BTR on top of a T5-base. 
Our experimental results showed that, by reranking candidates from a fully pre-trained and fine-tuned T5-base model, the BTR on top of the T5-base can achieve an $F_{0.5}$ score of 65.47 on the CoNLL-14 benchmark.
The BTR on top of T5-base also outperformed T5-base on the BEA test set by 0.76 points, achieving an $F_{0.5}$ score of 71.27.
\textcolor{black}{The BTR demonstrated its effectiveness in comparison to the post-combination-based reranker, CTR, particularly in scenarios involving token insertion and deletion within ungrammatical sentences generated by language-learners at the medium or advanced English levels.
In contrast to CTR and the encoder-based reranker, ETR-P, which exhibited flat probability distributions for reranking, the BTR exhibited a more dynamic behavior.}
Adopting negative sampling for the BTR %also
generated a peaked probability distribution for ranking. Consequently, grammatical suggestions could be selected by using $\lambda$.
\textcolor{black}{While the CTR struggled with predicting tokens at the beginning positions, the BTR treated each position with equal difficulty. Thus, the integration of bidirectional representation into the neural network structure aligns the BTR more closely with the characteristics of a cyclic directed graph, compared to the post-combination-based approach.}
Furthermore, the BTR on top of T5-base was robust even when reranking candidates from T5-large and improved the performance by 0.26 points on the BEA test set.
\textcolor{black}{These experimental results confirmed our two assumptions:}
\begin{quote}
\begin{itemize}
\item The utilization of the BERT-style self-attention mechanism could yield more useful bidirectional features, leading to superior results compared to the post-combination-based method.
\item Integrating a self-attention mechanism into the seq2seq architecture is more appropriate for the GEC task than incorporating it into the encoder-only architecture. This is because the seq2seq architecture has advantageous in capturing dependencies between source and target sequences.
\end{itemize}
\end{quote}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Work}

For directly predicting the target corrections from corresponding input tokens, \citeA{omelianchuk-etal-2020-gector} and \citeA{malmi-etal-2022-text} regarded the encoder of the transformer model as a non-autoregressive GEC sequence tagger.
The experimental results of \citeA{omelianchuk-etal-2020-gector} showed that, compared with the randomly initialized LSTM \cite{6795963}, the pre-trained models, such as RoBERTa \cite{liu2019roberta}, GPT-2 \cite{radford2019language}, and ALBERT \cite{lan2019albert}, can achieve higher $F_{0.5}$ scores as a tagger.
\citeA{sun2021instantaneous} considered GEC as a seq2seq task and introduced shallow aggressive decoding (SAD) for the decoder of the transformer. With the SAD, the performance of a pre-trained seq2seq model, BART, surpassed the sequence taggers of \citeA{omelianchuk-etal-2020-gector}.
The T5 xxl model is a pre-trained seq2seq model with 11B parameters \cite{2020t5}. After fine-tuning with the cLang-8 corpus, T5 xxl and mT5 xxl \cite{xue-etal-2021-mt5}, a multilingual version of T5, achieved state-of-the-art results on GEC benchmarks in four languages: English, Czech, German, and Russian \cite{rothe-etal-2021-simple}.
This demonstrated that performing a single fine-tuning step for a fully pre-trained seq2seq model is a simple and effective method for GEC without incorporating a copy mechanism \cite{zhao-etal-2019-improving}, the SAD, or the output from a pre-trained MLM \cite{kaneko-etal-2020-encoder}.
Despite the improvements owing to the pre-trained representations, the conventional seq2seq structure suffers from a prediction bias owing to its unidirectional decoding. According to \citeA{liu-etal-2021-neural}, by using beam search, a fully pre-trained seq2seq GEC model \cite{kiyono-etal-2019-empirical} can generate several high-quality grammatical hypotheses, including one that is more grammatical than the selected one.

To address the shortcoming of the unidirectional decoding, previous studies \cite{kiyono-etal-2019-empirical,kaneko-etal-2019-tmu,kaneko-etal-2020-encoder} introduced reversed representations to rerank the hypotheses. \citeA{kiyono-etal-2019-empirical} and \citeA{kaneko-etal-2020-encoder} utilized a seq2seq GEC model that decodes in the opposite direction (right-to-left) to rerank candidates, which was effective for selecting a more grammatical sentence than the original one. This finding motivated us to use a bidirectional decoding method for our model. 
Instead of using a seq2seq model, \citeA{kaneko-etal-2019-tmu} fine-tuned BERT as a reranker to evaluate the grammatical quality of a sentence.
By using masked language modeling, BERT learned deep bidirectional representations to distinguish between grammatical and ungrammatical sentences. However, BERT did not consider the positions of corrections, as it discarded the source sentence and considered only the target sentence.
This rendered it difficult for BERT, as a reranker, to recognize the most suitable corrected sentence for an ungrammatical sentence. \citeA{salazar-etal-2020-masked} proposed the use of pseudo-log-likelihood scores ($\mathrm{PLL}$) for reranking. 
They demonstrated that RoBERTa, using $\mathrm{PLL}$ for reranking, outperformed the conventional language model GPT-2 when reranking candidates in speech recognition and machine translation tasks.
\citeA{zhang-etal-2021-language} also claimed that the pre-trained model, MPNet \cite{song2020mpnet}, was more effective than GPT-2 when using $\mathrm{PLL}$ for reranking in discourse segmentation and parsing.

\citeA{zhang-van-genabith-2021-bidirectional} proposed a bidirectional transformer-based alignment \linebreak
(BTBA) model, which aims to assess the alignment between the source and target tokens in machine translation.
To achieve this, BTBA masked and predicted the current token with attention on both left and right sides of the target context to produce alignments for the current token. 
Specifically, to assess alignments from the attention scores in all cross-attention layers, the decoder in BTBA discarded the last feed-forward layer of the transformer model and directly predicted the masked tokens from the output of the last cross-attention layer. Although the target context on both sides was considered, one limitation of BTBA was that the computed alignments ignored the representation of the current token. 
To produce more accurate alignments, \citeA{zhang-van-genabith-2021-bidirectional} introduced full context based optimization (FCBO) for fine-tuning, wherein BTBA no longer masked the target sentence to use the full target context.

In this study, to determine the most appropriate correction for a given erroneous sentence, we modeled the BTR as a seq2seq reranker, which encoded the erroneous sentence using an encoder and decoded a corrected sentence using a decoder.
In contrast to the conventional seq2seq model, we use masked language modeling to mask and predict each target token in the decoder and estimate the sentence probability for each candidate using $\mathrm{PLL}$.
Moreover, unlike BTBA, the BTR preserved the last feed-forward layer in the decoder to predict masked tokens more accurately. Because the original data of the masked tokens should be invisible in the prediction, the FCBO fine-tuning step was not used in the BTR.
Compared with BTBA, the BTR maintained the structure of the transformer model and could easily inherit parameters from pre-trained models.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Bidirectional Transformer Reranker (BTR)}
\label{section:bidirectional_reranker}

For an ungrammatical sentence $\bm{x}$, a base GEC model first generated the top-$a$ corrected sentences \textcolor{black}{$\mathcal{Y}_{a}$}. We assumed $\bm{y}_{base} \in \mathcal{Y}_{a}$ was the top-ranked hypothesis from the base GEC model. The BTR used masked language modeling in the decoder to estimate the probability of each corrected sentence. Subsequently, it selected and accepted the most optimal corrected sentence $\bm{y}_{BTR}$ from $\mathcal{Y}_{a}$ based on the estimated sentence probability, as described in the following. Figure~\ref{fig:procedure} shows the overview of the BTR for the entire process.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 2
\begin{figure}[b]
\begin{center}
\includegraphics{31-1ia1f2.pdf}
\end{center}
\caption{Overview of the reranking procedure by using the bidirectional transformer reranker (BTR).}
\label{fig:procedure}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Generating Candidates}
\label{seq2seq_gec}

As a reranker, for a particular ungrammatical sentence $\bm{x}=(x_1, \ldots, x_n)$, where $x_i$ is the $i$-th token in $\bm{x}$, the BTR should compare all corresponding corrected sentences $\mathcal{Y}$ and select the most grammatical one. However, processing all possible corrected sentences is intractable, as suggested by \citeA{stahlberg-byrne-2019-nmt}, we considered a subset of sequences $\mathcal{Y}_{a}$ based on the top-$a$ results from a base GEC model.
Given an ungrammatical sentence $\bm{x}$, a GEC model corrects $\bm{x}$ into its grammatical sentence $\bm{y} = (y_1, \ldots, y_m)$, where $y_j$ is the $j$-th token in $\bm{y}$.
This seq2seq characteristic renders the seq2seq model, also known as the encoder-decoder model, particularly well-suited as the GEC model.
As an auto-regressive model, a seq2seq-based GEC model with parameter $\theta$ decomposes $p(\bm{y}|\bm{x};\theta)$ as follows: 
\begin{gather}
\log p(\bm{y}|\bm{x};\theta) \!=\!\! \sum_{j=1}^{m} \log p(y_j|\bm{x}, \bm{y}_{<j};\theta), \label{eq:auto_regressive}\\
 \textcolor{black}{p(y_j|\bm{x},\bm{y}_{<j};\theta) \!=\! \mathrm{softmax}(W\tilde{s}_{j}+b),}
\label{eq:linear_softmax}
\end{gather}
where $\tilde{s}_{j}$ is the final hidden state from the decoder at the $j$-th decoding step, $W$ is a weight matrix, $b$ is a bias term, and $\bm{y}_{<j}$ denotes $(y_1, \ldots, y_{j-1})$.

The current most popular seq2seq-style neural network is transformer, which comprises two primary components: encoder and decoder, as described below. The encoder calculates the hidden representations $\widetilde{H}$ of a source sentence $\bm{x}$, whereas the decoder generates the hidden states $\widetilde{S}$ to create a target sentence $\bm{y}$ from $\widetilde{H}$.
Let $\mathrm{FNN}$ denote a feed-forward layer and $\mathrm{Attn}(q, K, V )$ the attention layer, where $q$, $K$, and $V$ indicate the query, key, and value, respectively. Assuming the decoder comprises $L$ layers. To compute $\tilde{s}_{j}$, the encoder first encodes $\bm{x}$ into its representation $\widetilde{H}$ \textcolor{black}{(Eq.~(\ref{eq:Attns_bert}))}. Subsequently, for $\ell \in L$, the hidden state $\tilde{s}_{j}^{\ell}$ of the $\ell$-th layer in the decoder is computed using 
\begin{align}
  s_{j}^{\ell} &= \mathrm{Attn}_{s}(\tilde{s}_{j}^{\ell-1}, \widetilde{S}_{\leq{j}}^{\ell-1}, \widetilde{S}_{\leq{j}}^{\ell-1}), \label{eq:Attns_seq2seq} \\ 
  \hat{s}_{j}^{\ell} &= \mathrm{Attn}_{c}(s_{j}^{\ell}, \widetilde{H}, \widetilde{H}), \label{eq:attnc} \\ 
  \tilde{s}_{j}^{\ell} &= \mathrm{FNN}(\hat{s}_{j}^{\ell}), \label{eq:FFN}
\end{align}
where $\tilde{s}_{j}^{0}$ is the embedding of the token $y_{j-1}$ and $\tilde{s}_{1}$ is the state for the special token $<$s$>$ that indicates the start of a sequence. Further, $\widetilde{S}_{\leq{j}}^{\ell-1}$ denotes a set of hidden states $(\tilde{s}_{1}^{\ell-1},\ldots, \tilde{s}_{j}^{\ell-1})$, and $\mathrm{Attn}_{s}$ and $\mathrm{Attn}_{c}$ indicate the self-attention and cross-attention layers, respectively. A causal attention mask can be used to compute $S^{\ell}$ in parallel, as shown 
%%%%in Figure~\ref{fig:causal_attention}.
    in Figure~\ref{01fig03}a.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 3
\begin{figure}[b]
\begin{center}
\includegraphics{31-1ia1f3.pdf}
\end{center}
%%%%\label{fig:causal_attention}
%%%%\label{fig:fully_attention}
\hangcaption{Mask patterns in the transformer model (a) and in the BTR (b) for the self-attention mechanism in the decoder. Light cells indicate no attention.}
\label{01fig03}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\textcolor{black}{This study employed the pre-trained T5 model as the base GEC model, owing to its widely accepted transformer architecture and its ability to attain state-of-the-art results in GEC by using beam search for decoding \cite{rothe-etal-2021-simple}.}
However, previous studies \cite{li2016mutual,vijayakumar2017diverse} have suggested that beam search tends to generate sequences with slight differences.
This can constrain the upper bound score when reranking candidates \cite{ippolito-etal-2019-comparison}.
To select the optimal decoding method for this conventional transformer-based GEC Model, \textcolor{black}{CTGEC}, we compared beam search with diverse beam search \cite{vijayakumar2017diverse}, top-k sampling \cite{fan-etal-2018-hierarchical}, and nucleus sampling \cite{Holtzman2020The}.
For each pair of data in CoNLL-13 corpus \cite{ng-etal-2013-conll}, all decoding methods were required to generate 5 candidate sequences with a maximum sequence length of 200. When using diverse beam search, the beam group and diverse penalty were set at 5 and 0.4, respectively. Meanwhile, the top-k was set at 50 for top-k sampling, and the top-p was set at 0.95 for nucleus sampling.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[b]
\input{01table01.tex}%
\caption{Results for the \textcolor{black}{CTGEC} on the CoNLL-13 corpus with various decoding methods.}
\label{table:comparison_of_diverse_decoding_methods}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Table~\ref{table:comparison_of_diverse_decoding_methods} presents the comparison of results among different decoding methods.
Oracle indicates the upper bound score that can be achieved with the generated candidates. If the candidates included the correct answer, we assumed the prediction to be correct. Unique (\%) indicates the rate of unique sequences among all candidates. Gold (\%) indicates the rate of pairs of data whose candidates include the correct answer. The results show that beam search generates more diverse sentences with the highest Oracle score compared to nucleus sampling, top-k sampling, and diverse beam search. This may be because, in the GEC task, most of the tokens in the target were the same as the source, which caused a peaked probabilities distribution to focus on one or a small number of tokens. Consequently, a top-k filtering method such as beam search generates more diverse sentences than sampling or using probability as a diverse penalty.
Based on these results, we selected beam search as the decoding method for \textcolor{black}{CTGEC} during inference.
For evaluating \textcolor{black}{CTGEC}, it generated the top-ranked hypothesis with a beam size of 5.
To generate the top-$a$ candidates $\mathcal{Y}_{a} = \{\bm{y}_{1}, \ldots,\bm{y}_{a} \}$ for reranking, it generated the hypotheses with a beam size of $a$ and a maximum sequence length of 128 and 200 for the datasets in training and prediction, respectively.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Target Sentence Probability}
\label{section:sentence_prob}

\textcolor{black}{After generating candidates, the next step of the BTR involves extracting features from these candidates to re-estimate their sentence probabilities.
To guarantee the extraction of useful features, \citeA{10.5555/1756006.1953039} assumed that a good hidden state should exhibit stability and robustness when faced with corrupted input, while being capable of capturing the structure of the corrupted input to facilitate the recovery of the corresponding clean input. This is akin to a denoising task. Building upon this assumption, \citeA{10.5555/1756006.1953039} proposed the denoising autoencoder algorithm to train a stochastic operator $p(\bm{x}|\bm{x}_{\backslash \kappa})$ to reconstruct the input $\bm{x}$ from its corrupted version $\bm{x}_{\backslash \kappa}$ through self-supervised learning, where $\kappa$ denotes the set of corrupted positions in the input $\bm{x}$. An example is presented in Figure~\ref{fig:self_supervised_learning}.}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 4
\begin{figure}[b]
\begin{center}
\includegraphics{31-1ia1f4.pdf}
\end{center}
\hangcaption{Example of self-supervised learning, where $\bm{x}'$ represents the reconstructed text derived from $\bm{x}_{\backslash \kappa}$. The loss $\mathcal{L}(\bm{x}', \bm{x})$ quantifies the dissimilarity between the true text $\bm{x}$ and the predicted text $\bm{x}'$, and the model is optimized by minimizing the loss to reduce this discrepancy.}
\label{fig:self_supervised_learning}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Masked language modeling, which is used in BERT, was introduced to learn bidirectional representations for a given sentence $\bm{x}$ through self-supervised learning \cite{devlin-etal-2019-bert}. 
BERT follows the denoising autoencoder algorithm to utilize a masking noise strategy to corrupt a fraction of elements of $\bm{x}$. This resulted in a corrupted version $\bm{x}_{\backslash \kappa}$. Specifically, 15\% of the elements in $\bm{x}$ were randomly selected by the training data generator. If $x_{i}$ was chosen, it was replaced with (1) the mask token $<$M$>$ 80\% of the time, (2) a random token from vocabulary 10\% of the time, and (3) the unchanged $x_{i}$ 10\% of the time. An example of sentence corruption is presented in Table~\ref{table:masked_sentence_example}.
The model parameter $\theta$ \textcolor{black}{of BERT} was optimized by maximizing the following objective:
\begin{equation}
\log p(\bm{x}_{\kappa}|\bm{x}_{\backslash \kappa };\theta)\approx \sum_{k \in \kappa} \log p(x_{k}|\bm{x}_{\backslash \kappa};\theta).
\end{equation}
\textcolor{black}{The architecture of BERT comprised an encoder-only transformer model.} Similar to Eq.~(\ref{eq:linear_softmax}), a linear transformation with a softmax function was used for the final hidden state $\tilde{h}_{k}$ to predict $p(x_{k}|\bm{x}_{\backslash \kappa };\theta)$.
\textcolor{black}{BERT departs from the conventional design of transformer by discarding the decoder, focusing solely on the encoder. We assumed that the encoder comprised $L$ layers.}
Without the cross-attention, $\tilde{h}_{k}^{\ell}$ is the feed-forward result of $h_{k}^{\ell}$:
\begin{equation}
  h_{k}^{\ell} = \mathrm{Attn}_{s}(\tilde{h}_{k}^{\ell-1}, \widetilde{H}_{\backslash \kappa }^{\ell - 1}, \widetilde{H}_{\backslash \kappa }^{\ell - 1}), \label{eq:Attns_bert}
\end{equation}
where $\tilde{h}_{k}^{0}$ is the embedding of the $k$-th token in $\bm{x}_{\backslash \kappa}$ and $\widetilde{H}_{\backslash \kappa }^{\ell-1}=(\tilde{h}_{1}^{\ell-1}, \ldots,\tilde{h}_{m}^{\ell-1})$ denotes a set of hidden states for $\bm{x}_{\backslash \kappa}$. 
\textcolor{black}{This formula results in BERT being a cyclic directed graph. $h_{k}^{\ell}$ utilized both the left and right sides of the context of the masked token $x_{k}$ to capture deeper representations. This distinguishes it from $s_{j}^{\ell}$ in Eq.~(\ref{eq:Attns_seq2seq}) that only employs the left side of the context as an acyclic directed graph \cite{yamakoshi-etal-2022-probing}.}
The corresponding $\mathrm{PLL}$ for $\bm{x}$ is computed by
\begin{equation}
\mathrm{PLL}(\bm{x};\theta) :=\sum_{i=1,\kappa=\{i\}}^{|\bm{x}|}\log p(x_i|\bm{x}_{\backslash \kappa};\theta ),
\end{equation}
where $|\bm{x}|$ is the length of $\bm{x}$. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 2
\begin{table}[t]
\input{01table02.tex}%
\caption{\textcolor{black}{Example of a sentence of length 11 before and after corruption.}}
\label{table:masked_sentence_example}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\textcolor{black}{As depicted in Figure~\ref{fig:directed_graphs}, cyclic directed graphs can leverage more information than acyclic directed graphs for predictions. Thus, we assumed that decomposing the conditional sentence probability of a sequence $\bm{y}$ using a BERT-style masked language model could yield more useful features than auto-regressive models (i.e., Eq.~(\ref{eq:auto_regressive})).
Post-combining two opposite-directional seq2seq models, such as $p(y_j|\bm{x}, \bm{y}_{<j};\theta)$ and $p(y_j|\bm{x}, \bm{y}_{>j};\theta)$, can indeed form a cyclic directed graph and has demonstrated effectiveness in prior research \cite{kiyono-etal-2019-empirical,kaneko-etal-2020-encoder}. However, this direct combination might result in the loss of deep information. In contrast, considering that the multi-head self-attention mechanism was originally designed to extract deep features from different aspects, we assumed that the BERT-style self-attention mechanism as Eq.~(\ref{eq:Attns_bert}) could yield more useful bidirectional features than the post-combination-based method.}
\textcolor{black}{Furthermore,} as suggested by \citeA{salazar-etal-2020-masked}, when using $\mathrm{PLL}$ to estimate the cross-entropy loss, the loss of $x_i|\bm{x}_{\backslash \kappa}$ versus $i$ from BERT is flatter than GPT-2, wherein the chain rule is used. As the candidate sentences might have different lengths, $\mathrm{PLL}$ is ideal for reranking.
\textcolor{black}{Consequently}, we decomposed the conditional sentence probability of $\bm{y}$ as:
\begin{equation}
\log p(\bm{y}|\bm{x}; \theta) \approx \mathrm{PLL}(\bm{y}|\bm{x}; \theta) = \!\!\!\!\!\!\! \sum_{j=1,\kappa=\{j\}}^{|\bm{y}|} \!\!\!\!\!\! \log p(y_j|\bm{x}, \bm{y}_{\backslash \kappa};\theta).
\end{equation}
\textcolor{black}{Assuming that the seq2seq architecture is more suitable for the GEC task to handle two sequences than the encoder-only architecture, the BTR also adopted the seq2seq-style transformer architecture to estimate $p(y_j|\bm{x}, \bm{y}_{\backslash \kappa};\theta)$.}
As in Eq.~(\ref{eq:linear_softmax}), a linear transformation with the softmax function was utilized for the final hidden state $\tilde{s}_{j}$ to predict $p(y_j|\bm{x}, \bm{y}_{\backslash \kappa};\theta)$. 
$\tilde{s}_{j}$ is the result of $s_{j}$ following the cross-attention and feed-forward layers. \textcolor{black}{We assumed that the decoder comprises $L$ layers.} To capture the bidirectional representation, for $\ell \in L$, \textcolor{black}{the BTR} computes $s_{j}^{\ell}$ as:
\begin{equation}
s_{j}^{\ell} = \mathrm{Attn}_{s}(\tilde{s}_{j}^{\ell-1}, \widetilde{S}_{\backslash \kappa }^{\ell-1}, \widetilde{S}_{\backslash \kappa }^{\ell-1}),
\end{equation}
where $\tilde{s}_{j}^{0}$ is the embedding of the $j-1$-th word in $\bm{y}_{\backslash \kappa}$, $\tilde{s}_{1}$ is the state-of-the-start token $<$s$>$, and $\widetilde{S}_{\backslash \kappa }^{\ell-1}=(\tilde{s}_{1}^{\ell-1}, \ldots,\tilde{s}_{m+1}^{\ell-1})$ denotes a set of hidden states for the joint sequence of $<$s$>$ and $\bm{y}_{\backslash \kappa}$.
%%%%Figure~\ref{fig:fully_attention} shows the proposed fully-visible attention mask for computing $S^{\ell}$ in parallel.
    Figure~\ref{01fig03} shows the proposed fully-visible attention mask for computing $S^{\ell}$ in parallel.
The procedure of using the BTR to predict $p(y_j|\bm{x}, \bm{y}_{\backslash \kappa};\theta)$ is shown in Figure~\ref{fig:bi_transformer}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 5
\begin{figure}[t]
\begin{center}
\includegraphics{31-1ia1f5.pdf}
\end{center}
\hangcaption{Bidirectional transformer architecture. The left and right columns indicate the encoder and decoder, respectively. The self-attention mechanism in the decoder utilizes the fully-visible mask (Figure \ref{01fig03}b), unlike the conventional transformer.}
\label{fig:bi_transformer}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Objective Function}
\label{objective_function}

Let $\boldsymbol y_{gold} \in \mathcal{Y}$ denote the gold correction for $\boldsymbol x$.
For $\boldsymbol y \in \mathcal{Y}_{a} \cup \{\boldsymbol y_{gold}\} $, we followed the setting of BERT to randomly mask 15\% of $\boldsymbol y$ and denoted $\kappa$ as the set of masked positions. Consequently, the distribution of the masked tokens satisfied the 8:1:1 masking strategy. 
\textcolor{black}{Notably, each time a training example $\bm{y}$ was presented, a different corrupted version $\bm{y}_{\backslash \kappa}$ was generated.}
Following previous research \cite{welleck-etal-2019-dialogue,zhang-etal-2021-language,song-etal-2021-bob}, considering the masked sentence $\bm{y}_{\backslash \kappa }$, the model parameter $\theta$ of the BTR was optimized by maximizing the likelihood and minimizing the unlikelihood as:
\begin{equation}
\log p(\bm{y}_{\kappa}|\bm{x},\bm{y}_{\backslash \kappa };\theta) \approx \sum_{k \in \kappa}[\underbrace{\mathbbm{1}_{\boldsymbol y} \log p(y_k|\bm{x}, \bm{y}_{\backslash \kappa };\theta)}_{\text{likelihood}} + \underbrace{(1-\mathbbm{1}_{\boldsymbol y})\log (1 - p(y_k|\bm{x}, \bm{y}_{\backslash \kappa };\theta))}_{\text{unlikelihood}}],     \label{eq:maxmize_likelyhood}
\end{equation}
where $ p(y_k|\bm{x}, \bm{y}_{\backslash \kappa };\theta)$ was computed as in Section~\ref{section:sentence_prob}. Further, $\mathbbm{1}_{\boldsymbol y}$ is an indicator function defined as follows:
\begin{equation}
\mathbbm{1}_{\boldsymbol y} := \begin{cases}
1 & \text{if } \boldsymbol y=\boldsymbol y_{gold}    \\
0 & \text{if } \boldsymbol y \neq \boldsymbol y_{gold}
\end{cases}.
\end{equation}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Inference}
\label{sec:inference}

For $\boldsymbol y \in \mathcal{Y}_{a}$, the BTR scores $\bm{y}$ using
\begin{equation}
f(\bm{y}|\bm{x}) =\frac{\exp (\mathrm{PLL}(\bm{y}|\bm{x};\theta)/|\bm{y}|)}{\sum_{\bm{y}' \in \mathcal{Y}_{a}} \exp(\mathrm{PLL}(\bm{y}'|\bm{x};\theta)/|\bm{y}'|)}.
\end{equation}
Hereafter, we denote $\bm{y}_{BTR} \in \mathcal{Y}_{a}$ as the candidate with the highest score $f(\bm{y}_{BTR}|\bm{x})$ for given $\bm{x}$ in the BTR. Here, $f(\bm{y}|\bm{x})$ was also considered to indicate the confidence of the BTR. Because the BTR was optimized with Eq.~(\ref{eq:maxmize_likelyhood}), a high score for $\bm{y}_{BTR}$ indicated a confident prediction and a low score indicated an unconfident prediction, \textcolor{black}{as proven in Section~\ref{sec:effectiveness_of_lambda}}. 

Considering that we constructed the base GEC model from a fully pre-trained seq2seq model and the BTR from an insufficiently pre-trained model, we introduced an acceptance threshold \textcolor{black}{$\lambda$} to decide whether to accept the suggestion from the BTR.
We accepted $\bm{y}_{BTR}$ only when it satisfied the following equation; otherwise, $\bm{y}_{base}$ was the final result: 
\begin{equation}
f(\bm{y}_{BTR}|\bm{x}) - f(\bm{y}_{base}|\bm{x}) > \lambda,
\label{eq:accpet_threshold}
\end{equation}
where $\lambda$ is a hyperparameter tuned on the validation data.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experiments}

\textcolor{black}{To evaluate the effectiveness of our proposal, we first investigated the impact of the proposed seq2seq-styled self-attention-based cyclic directed graph by comparing it with rerankers that utilized the encoder-styled self-attention-based cyclic directed graph (ETR-C and ETR-P) and the seq2seq-styled post-combination-based cyclic directed graph (CTR).
We assessed model performances across four GEC datasets using the English language. In addition, to assess the performance under various scenarios, we conducted fine-grained evaluations across distinct error types, token positions, candidate qualities, and levels of language-learners. Furthermore, to demonstrate our claim in Section~\ref{sec:inference}, that is \emph{$f(\bm{y}|\bm{x})$ is also considered to indicate the confidence of the BTR}, we performed experiments to determine the efficacy of incorporating $\lambda$ in Eq.~(\ref{eq:accpet_threshold}).}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Compared Methods}
\label{sec:compared_methods}

We evaluated the BTR as a reranker for two versions of candidates: normal and high-quality ones, generated by two seq2seq GEC models, \textcolor{black}{CTGEC} and \textcolor{black}{CTGEC} (large).
We compared the BTR with three other rerankers: \textcolor{black}{CTR}, \textcolor{black}{ETR-C}, and \textcolor{black}{ETR-P}.

\textbf{\textcolor{black}{CTGEC}}: We used the state-of-the-art model \cite{rothe-etal-2021-simple} as our base model for GEC. This base model inherited the pre-trained T5 version 1.1 model (T5-base) \cite{2020t5} and was fine-tuned as described in Section~\ref{seq2seq_gec}. 
\textcolor{black}{We denoted this base model with conventional transformer architecture as CTGEC hereafter.}
Although the T5 xxl model yielded the most grammatical sentences in \citeA{rothe-etal-2021-simple}, it contained 11B parameters and was not suitable for our current experimental environment. Thus, we modeled \textcolor{black}{CTGEC} on top of a 248M-parameter T5-base model.
To reproduce the experimental results of \citeA{rothe-etal-2021-simple}, we followed their setting and performed fine-tuning \textcolor{black}{CTGEC} using the cLang-8 dataset. 

\textbf{\textcolor{black}{CTGEC} (large)}: To investigate the potential of the BTR for reranking high-quality candidates, we also fine-tuned one larger \textcolor{black}{CTGEC} model with a 738M-parameter T5-large structure. We denote this model as \textcolor{black}{CTGEC} (large). 

\textbf{\textcolor{black}{CTR}}: The decoder of the conventional seq2seq model can generate a target sentence either in a left-to-right \textcolor{black}{(L2R)} or right-to-left \textcolor{black}{(R2L)} direction. Because \textcolor{black}{CTGEC} utilized the \textcolor{black}{L2R} direction, and previous studies \cite{sennrich-etal-2016-edinburgh,kiyono-etal-2019-empirical,kaneko-etal-2020-encoder} have demonstrated the effectiveness of reranking using the \textcolor{black}{R2L} model, we followed \citeA{kaneko-etal-2020-encoder} to construct four \textcolor{black}{R2L directional CTGEC} models for reranking, which we denote as CTR. CTR reranked candidates based on the sum score of the base model (CTGEC) and ensembled CTR, which could be regarded as a seq2seq-styled post-combination-based cyclic directed graph.

\textbf{ETR-C}: We followed \citeA{kaneko-etal-2019-tmu} to fine-tune four BERT with 334M parameters as encoder-styled self-attention-based cyclic directed graphs for comparisons with the proposed seq2seq-styled model. During fine-tuning, both the source and target sentences were annotated with either $<$0$>$ (ungrammatical) or $<$1$>$ (grammatical) label for BERT for classification. During inference, the ensembled BERT reranked candidates based on the predicted score for the $<$1$>$ label.
We denote this encoder-only transformer binary-classification-based reranker as ETR-C.

\textbf{ETR-P}: We \textcolor{black}{also} fine-tuned four 125M parameters RoBERTa \textcolor{black}{as encoder-styled self-attention-based cyclic directed graphs}. During fine-tuning, the source and target sentences were concatenated, and RoBERTa masked and predicted only the target sentence as the BTR. During prediction, the ensembled RoBERTa reranked candidates with the acceptance threshold $\lambda$ as the BTR.
We denote this encoder-only transformer PLL-based reranker as ETR-P.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Setup for the BTR}
\label{sec:setup_for_the_BTR}

Owing to the lack of a pre-trained seq2seq model with a self-attention mechanism for masked language modeling in the decoder, we constructed the BTR using the 248M T5 model (T5-base) and pre-trained it with the Realnewslike corpus \cite{NEURIPS2019_3e9f0fc9}. 
For comparisons with the CTR, we also constructed the CTR using a T5-base, and pre-trained BTR and CTR as follows.
To accelerate pre-training, we initialized the \textcolor{black}{BTR and the CTR' model-parameters} with the fine-tuned parameters $\theta$ of \textcolor{black}{CTGEC}.
During pre-training, we followed \citeA{2020t5} for self-supervised learning with a span masking strategy. Specifically, 15\% of the tokens in a given sentence were randomly sampled and removed. The input sequence was constructed using the rest tokens, whereas the target sequence was the concatenation of dropped-out tokens. 
An example is provided in Table~\ref{table:example_data}.
We pre-trained the \textcolor{black}{BTR and the CTR} with $65536=2^{16}$ and 10000 steps, respectively. Because the BTR masked and predicted only 15\% of the tokens in Eq.~(\ref{eq:maxmize_likelyhood}), the true steps for the BTR were $2^{16}\times 0.15 \approx 10000$. We used a maximum sequence length of 512 and a batch size of $2^{20}=1048576$ tokens. We pre-trained $10000 \times 2^{20} \approx 10.5$B tokens, which were less than the pre-trained T5 with 34B tokens. The pre-training for \textcolor{black}{the CTR} and the BTR required 2 and 13 days, respectively, using 2 NVIDIA A100 80GB GPUs. This indicates the BTR requires more training time and resources than \textcolor{black}{the CTR}. Figure~\ref{fig:pre-training_loss} show a plot of the pre-training loss. As evident, the training loss of \textcolor{black}{the CTR} suddenly dropped from 1.48 to 1.3 after the first epoch (7957 steps).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[t]
\input{01table03.tex}%
\hangcaption{Examples of data pairs for self-supervised and supervised learning used by each model. The grammatical text is ``Thank you for inviting me to your party last week.'' $<$M$>$ denotes a mask token, $<$X$>$, $<$Y$>$, and $<$Z$>$ denote sentinel tokens that are assigned unique token IDs, and $<$1$>$ denotes the input sentence is classified as a grammatical sentence. \textcolor{red}{\textbf{Bold}} indicates an error in the source sentence while \textcolor{blue}{\underline{Underline}} indicates a token randomly replaced by the BERT-style masking strategy.}
\label{table:example_data}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

After pre-training, we successively fine-tuned the CTR and the BTR with the cLang-8 dataset. Similar to that in \textcolor{black}{CTR}, \textcolor{black}{ETR-C}, and \textcolor{black}{ETR-P}, our fine-tuned BTR was the ensemble of four models with random seeds.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 6
\begin{figure}[b]
\begin{center}
\includegraphics{31-1ia1f6.pdf}
\end{center}
\vspace{-0.5\Cvs}
\caption{Pre-training loss for R2L (left) and the BTR (right).} 
\label{fig:pre-training_loss}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Datasets}

To assess the performance of our proposal, \pagebreak
we conducted experiments on four publicly available datasets: CoNLL-13, CoNLL-14 \cite{ng2014conll}, BEA \cite{bryant-etal-2019-bea}, and JFLEG \cite{napoles-etal-2017-jfleg}, all of which are in English. For a fair comparison, we pre-trained \textcolor{black}{the CTR} and the BTR with the Realnewslike corpus. This corpus contains 37~GB of text data and is a subset of the C4 corpus \cite{2020t5}. To shorten the input and target sequences, we split each text into paragraphs. 
During fine-tuning, we followed the steps of \citeA{rothe-etal-2021-simple} and regarded the cLang-8 corpus as the training dataset. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 4
\begin{table}[b]
\input{01table04.tex}%
\caption{\textcolor{black}{Dataset sizes.}}
\label{table:data_distribution}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

While the CoNLL-13 dataset was used for validation, the standard benchmarks from JFLEG, CoNLL-14, and the BEA test set were used for evaluation.
The CoNLL-14 corpus considers the minimal edit of corrections, whereas, JFLEG evaluates the fluency of a sentence. Further, the BEA corpus contains considerably more diverse English language levels and domains than the CoNLL-14 corpus. Each sentence in the BEA test set is classified into either A (beginner), B (intermediate), C (advanced), or N (native) corresponding to the Common European Framework of Reference for Languages (CEFR) level \cite{council2001common}.
We used a cleaned version of CoNLL-13 with consistent punctuation tokenization styles. Section~\ref{sec:appendix_cleaning_for_conll_corpus} lists our cleaning steps and the experimental results on the cleaned CoNLL-14 set.
Table~\ref{table:data_distribution} summarizes the data statistics. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Evaluation Metrics}

The evaluation on the BEA test set was automatically executed in the official BEA-19 competition in terms of span-based correction $F_{0.5}$ using the ERRANT \cite{bryant-etal-2017-automatic} scorer. For the CoNLL-13 and 14 benchmarks, we evaluated the correction $F_{0.5}$ using the official $M^2$ \cite{dahlmeier-ng-2012-better} scorer. For the JFLEG corpus, we evaluated the GLEU \cite{napoles-etal-2015-ground}.

This paper presents only the significant results on the CoNLL-14 set, because the gold data for the BEA test set was unavailable, and the evaluation metric GLEU for the JFLEG test set required a sampling strategy for multiple references. We used the paired $t$-test to evaluate whether the difference between $\bm{y}_{BTR}$ and $\bm{y}_{base}$ on the CoNLL-14 set was significant, as only limited $\bm{y}_{BTR}$ differed from $\bm{y}_{base}$ among the suggestions from the BTR.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Hyperparameters}

Appendix~\ref{sec:appendix_hyperparameters} lists our hyperparameter settings for pre-training and fine-tuning each model.

We followed the setting presented by \citeA{zhang-etal-2021-language} to separately tune $a$ for training and prediction, based on the model performance on the validation dataset with candidates generated by \textcolor{black}{CTGEC}. We denote $a$ for training and prediction as $a_{train}$ and $a_{pred}$, respectively. 
The threshold ($\lambda$) for the BTR and \textcolor{black}{ETR-P} was tuned together with $a$.
We set $a_{train}$ to 20 and 0 for the BTR and \textcolor{black}{ETR-P}, respectively, and $a_{pred}$ was set to 5 for all rerankers.
When $a_{train}=20$, $\lambda$ was set to 0.4 and 0.8 with respect to the candidates generated by \textcolor{black}{CTGEC} and \textcolor{black}{CTGEC} (large), respectively.
When $a_{train}=0$, $\lambda$ for the \textcolor{black}{ETR-P} was set to 0.1 for the two versions of candidates.
The experimental results for tuning $a_{train}$, $a_{pred}$, and $\lambda$ are listed in Appendix~\ref{sec:appendix_candidates}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 5
\begin{table}[t]
\input{01table05.tex}%
\hangcaption{Results for the models on each dataset with candidates from \textcolor{black}{CTGEC}. \textbf{Bold} scores indicate the highest (p)recision, (r)ecall, $F_{0.5}$, and GLEU for each dataset. \underline{Underline} marks the second-highest scores.}
\label{table:main_result}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 6
\begin{table}[t]
\input{01table06.tex}%
\hangcaption{The mean $\pm$ std results on each dataset with candidates from \textcolor{black}{CTGEC}. \textbf{Bold} scores indicate the highest mean for each dataset. \underline{Underline} marks the second-highest mean scores.}
\label{table:mean_std}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Results}

Table~\ref{table:main_result} presents our main results.\footnote{The mean and standard deviation results of the BTR, \textcolor{black}{CTR}, and \textcolor{black}{ETR-P} are listed in Table~\ref{table:mean_std}.}
While reranking by the CTR yielded the highest $F_{0.5}$ score of 71.42 on the BEA test set, it yielded scores lower than that of the BTR on the CoNLL-14 and JFLEG test sets.
Meanwhile, the improvements induced by the CTR depended on the beam searching score from CTGEC. This suggested that the post-combination-based bidirectional representation offered fewer gains compared to the self-attention-based bidirectional representation from the BTR.
Reranking candidates by \textcolor{black}{ETR-C} resulted in the lower $F_{0.5}$ and GLEU scores than that using \textcolor{black}{CTGEC}. This may be because \textcolor{black}{ETR-C} considered only the target sentence and ignored the relationship between the source and the target.
The \textcolor{black}{BTR} %($\lambda=0.4$)
achieved an $F_{0.5}$ score of 71.27 on the BEA test set.
On the CoNLL-14 test set, the \textcolor{black}{BTR} %($\lambda=0.4$)
attained the highest $F_{0.5}$ score of 65.47, with improvements of 0.36 points from \textcolor{black}{CTGEC}.
The use of the threshold and negative candidates played an important role in the BTR. 
Without these two mechanisms, the BTR achieved scores of only 59.48 and 63.60 $F_{0.5}$ on the CoNLL-14 and BEA test sets, respectively, which were lower than those of the original selection.
In the meantime, the BTR without the threshold achieved the highest GLEU score of 59.52 on the JFLEG corpus, which indicates $\lambda=0.4$ is excessively high for the JFLEG corpus.
This is attributed to the different distributions and evaluation metrics between the CoNLL-13 and JFLEG corpus, as proven in Section~\ref{sec:effectiveness_of_lambda}.
Compared to \textcolor{black}{ETR-P} %($\lambda=0.1$)
w/o $a_{train}$ of the encoder-only structure, the \textcolor{black}{BTR} %($\lambda=0.4$)
could achieve higher $F_{0.5}$ scores on CoNLL-13, 14, and BEA test sets, and a competitive GLEU score on the JFLEG corpus. These results show the benefit of using the transformer with the encoder-decoder architecture in the BTR.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 7
\begin{table}[b]
\input{01table07.tex}%
\hangcaption{Results for the top five error types on the BEA test set. \textbf{Bold} scores indicate the highest $F_{0.5}$ for each error type. \underline{Underline} marks the second-highest scores.}
\label{table:error_type_result}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 8
\begin{table}[b]
\input{01table08.tex}%
\hangcaption{Results for each operation type with classified CEFR levels on the BEA test set with candidates from \textcolor{black}{CTGEC}. Edit operations are divided into \emph{Missing}, \emph{Replacement}, and \emph{Unnecessary} corresponding to inserting, substituting, and deleting tokens, respectively. \textbf{Bold} scores indicate the highest $F_{0.5}$ for each operation with the corresponding level. \underline{Underline} marks the second-highest scores.}
\label{table:Level_Error_results}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Evaluation on CEFR Levels and Error Types}

Tables~\ref{table:error_type_result} and \ref{table:Level_Error_results} present further details for different error types and CEFR levels. 
As shown in Table~\ref{table:error_type_result}, the \textcolor{black}{BTR} %($\lambda=0.4$) 
improved \textcolor{black}{CTGEC} for all top-5 error types.
\textcolor{black}{While the CTR achieved the highest $F_{0.5}$ scores for three error types: \emph{DET}, \emph{PREP}, and \emph{SPELL}, among the three models, its performance as a reranker resulted in lower $F_{0.5}$ scores for \emph{PUNCT} and \emph{ORTH} error types compared to CTGEC.}
\textcolor{black}{As shown in Table~\ref{table:Level_Error_results},} compared with A (beginner) level sentences, the BTR was more effective for B (intermediate), C (advanced), and N (native) level sentences. 
\textcolor{black}{Furthermore, in comparison to the CTR model, the BTR exhibited stronger performance in handling \emph{Missing} and \emph{Unnecessary} tokens; although the performance with \emph{Replacement} tokens was low. While the CTR achieved the highest $F_{0.5}$ scores for sentences at the beginner and native levels, the BTR attained the highest $F_{0.5}$ scores for sentences at the intermediate and advanced levels. This observation indicates the potential of applying the BTR to GEC for language-learners.}
%Furthermore, the BTR ($\lambda=0.4$) could effectively handle \emph{Missing} and \emph{Unnecessary} tokens but not \emph{Replacement} for the native sentences. 
It was more difficult to correct the \emph{Replacement} and \emph{Unnecessary} operations in the native sentences for all three models compared with the advanced sentences. 
This may be attributed to the writing style of native speakers being natural and difficult to correct with limited training data, whereas language learners may tend to use a formal language to render the correction easier. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Reranking High-quality Candidates}

Table~\ref{table:T5GEC_large_result} lists the performances when reranking high-quality candidates.
While \textcolor{black}{the CTR} still achieved the highest $F_{0.5}$ score on the BEA test set, it was less effective than the BTR on the JFLEG corpus.
Although the \textcolor{black}{BTR} %($\lambda=0.8$) 
used only 248M parameters and was trained with the candidates generated by \textcolor{black}{CTGEC}, it reranked candidates from \textcolor{black}{CTGEC} (large) and achieved score of 61.97 GLEU and 72.41 $F_{0.5}$ on the JFLEG and BEA test sets, respectively. This finding indicates the sizes of the BTR and the base model need not to be consistent, and a smaller BTR can also function as a reranker for a larger base model. 
\textcolor{black}{ETR-P} %($\lambda=0.1$) w/o $a_{train}$ 
achieved the highest $F_{0.5}$ score of 66.85 on the CoNLL-14 corpus with only 0.02-point improvement from \textcolor{black}{CTGEC} (large), which reflects the difficulty in correcting uncleaned sentences.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 9
\begin{table}[b]
\input{01table09.tex}%
\hangcaption{Results for the models on each dataset with candidates generated by \textcolor{black}{CTGEC} (large). \textbf{Bold} scores indicate the highest (p)recision, (r)ecall, $F_{0.5}$, and GLEU for each dataset. \underline{Underline} marks the second-highest scores.}
\label{table:T5GEC_large_result}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Evaluation on the Cleaned CoNLL Corpus}
\label{sec:appendix_cleaning_for_conll_corpus}

The original texts of CoNLL-13 and 14 contain several styles of \pagebreak
punctuation tokenization, such as ``DementiaToday,2012'' and ``known , a''. While these punctuation styles with/without spaces are not considered grammatical errors by a human, they are often identified as errors by automatic GEC scorers.
Moreover, although most of the sequences in CoNLL-14 are of sentence-level, several sequences are of paragraph-level owing to the punctuation without spaces. 
In this study, we cleaned the texts of CoNLL-13 and 14 using the ``en\_core\_web\_sm'' tool in spaCy \cite{honnibal2020spacy} such that all punctuation included spaces. The paragraph-level sequences were split into sentences with respect to the position of full stops. The cleaned CoNLL-14 corpus contained 1326 pairs of data.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 10
\begin{table}[b]
\input{01table10.tex}%
\hangcaption{Results for the models on the cleaned CoNLL-14 corpus with candidates from \textcolor{black}{CTGEC}. \textbf{Bold} scores indicate the highest precision, recall, and $F_{0.5}$. \underline{Underline} marks the second-highest scores.}
\label{table:main_result_conll14}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 11
\begin{table}[b]
\input{01table11.tex}%
\hangcaption{The mean $\pm$ std results on the cleaned CoNLL-14 corpus with candidates from \textcolor{black}{CTGEC}. \textbf{Bold} and \underline{Underline} scores indicate the highest and the second-highest means, respecively.}
\label{table:mean_std_conll14}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Tables~\ref{table:main_result_conll14}, \ref{table:mean_std_conll14}, and \ref{table:T5GEC_large_result_conll14} present the experimental results for the cleaned CoNLL-14 corpus. 
Compared to the results obtained for the original CoNLL-14 corpus (Tables~\ref{table:main_result} and \ref{table:T5GEC_large_result}), the Oracle performance on the cleaned version facilitated higher precision and competitive recall scores, resulting in an improved $F_{0.5}$ score. The score distributions on the cleaned corpus among the rerankers were similar to those observed in the original corpus. In particular, the \textcolor{black}{BTR} %($\lambda=0.4$) 
maintained its position as the top-performing reranker, achieving the highest $F_{0.5}$ scores of 70.00 on the cleaned corpus using candidates from \textcolor{black}{CTGEC}. Meanwhile, in contrast to the results on the original corpus, the \textcolor{black}{BTR} %($\lambda=0.8$) 
achieved the highest $F_{0.5}$ score of 71.20 when employing candidates from \textcolor{black}{CTGEC} (large), thereby emphasizing the significance of using cleaned text in the process.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 12
\begin{table}[b]
\input{01table12.tex}%
\hangcaption{Results for the models on the cleaned CoNLL-14 corpus with candidates from \textcolor{black}{CTGEC} (large). \textbf{Bold} scores indicate the highest precision, recall, and $F_{0.5}$. \underline{Underline} marks the second-highest scores.}
\label{table:T5GEC_large_result_conll14}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 13
\begin{table}[b]
\input{01table13.tex}%
\hangcaption{Results for the BTR ($\lambda=0.4$) on CoNLL-14 with candidates from \textcolor{black}{CTGEC}. $\bm{y}_{base}$ and $\bm{y}_{BTR}$ denote the selections by \textcolor{black}{CTGEC} and suggestions by the BTR, respectively. $\dagger$ indicates that the difference between $\bm{y}_{BTR}$ and $\bm{y}_{base}$ is significant with a $p\text{-value} < 0.05$. Bold scores represent the highest $F_{0.5}$ for each case.}
\label{table:pie_scores}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 7
\begin{figure}[b]
\begin{center}
\includegraphics{31-1ia1f7.pdf}
\end{center}
\hangcaption{Precision and recall of BTR ($a_{train}=20, a_{pred}=5$) with respect to different $\lambda$ on the CoNLL-13 set.}
\label{fig:all_versus_lambda_conll}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 8
\begin{figure}[p]
\begin{center}
\includegraphics{31-1ia1f8.pdf}
\end{center}
\hangcaption{Precision and recall of BTR ($a_{train}=10, a_{pred}=5$) with respect to different $\lambda$ on the BEA dev set.}
\label{fig:all_versus_lambda_bea}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 9
\begin{figure}[p]
\begin{center}
\includegraphics{31-1ia1f9.pdf}
\end{center}
\caption{GLEU of BTR ($a_{train}=5, a_{pred}=15$) with respect to different $\lambda$ on the JFLEG dev set.}
\label{fig:all_versus_lambda_jfleg}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Role of $\lambda$}
\label{sec:effectiveness_of_lambda}

Table~\ref{table:pie_scores} indicates the effect of using $\lambda$.
\emph{Equal} indicates that the suggestion $\bm{y}_{BTR}$ is exactly $\bm{y}_{base}$, \emph{Accept} indicates that $\bm{y}_{BTR}$ satisfies Eq.~(\ref{eq:accpet_threshold}) and $\bm{y}_{BTR}$ will be the final selection, while \emph{Reject} indicates that $\bm{y}_{BTR}$ does not satisfy the equation and $\bm{y}_{base}$ is still the final selection. 
Most of the final selections belonged to \emph{Equal} and achieved the highest $F_{0.5}$ score of 68.78.
This indicates that the sentences in \emph{Equal} can be corrected easily using both the \textcolor{black}{BTR} %($\lambda=0.4$) 
and \textcolor{black}{CTGEC}. 
Approximately $1/3$ of the new suggestions proposed by the \textcolor{black}{BTR} %($\lambda=0.4$) 
were accepted and achieved an $F_{0.5}$ score of 63.97, which was a 2.3-point improvement from $\bm{y}_{base}$.
However, approximately $2/3$ of the new suggestions were not accepted, and the original selection by \textcolor{black}{CTGEC} resulted in a higher $F_{0.5}$ score than these rejected suggestions.
These results show that, among the new suggestions, the BTR was confident only for certain suggestions. The confident suggestions tended to be more grammatical, whereas the unconfident suggestions tended to be less grammatical than the original selections.
To investigate the role of $\lambda$, we also analyzed the relationship between $\lambda$ and the corresponding precision, recall, and GLEU scores.\footnote{\textcolor{black}{We exclusively report the results for the dev set, as the trends observed in both the dev and test sets were the same. The results for the test set can be found in Appendix~\ref{sec:appendix_role_of_lambda}.}}
Figure~\ref{fig:all_versus_lambda_conll} shows the performance of the BTR ($a_{train}=20, a_{pred}=5$) on the \textcolor{black}{CoNLL-13} %and 14 corpus, respectively
corpus. With increase in $\lambda$, the acceptance rate, that is, the percentage of suggestions that the BTR accepted, decreased while the precision and recall for the \emph{Accept} suggestions increased.
This demonstrates our assumption (Section~\ref{sec:inference}) that the value of $f(\bm{y}|\bm{x})$ indicates the confidence of the BTR. It also reaffirms our previous findings, indicating that suggestions with higher confidence tended to be more grammatical than the original selections.
Regarding the entire corpus, when $\lambda=0.7$, this BTR achieved lower precision and recall score than $\lambda=0.4$ owing to the limited amount of $Accept$ suggestions.
Figure~\ref{fig:all_versus_lambda_bea} shows the performance of BTR ($a_{train}=10, a_{pred}=5$) on \textcolor{black}{the BEA dev} %and test corpus, respectively
corpus. In Figure~\ref{fig:all_versus_lambda_bea}, the BTR exhibits a performance similar to that on the \textcolor{black}{CoNLL-13}, where a larger $\lambda$ resulted in higher precision and recall for $Accept$ suggestions.
However, the performance over the whole corpus was also dependent on the acceptance rate. As shown in Figure~\ref{fig:all_versus_lambda_jfleg}, the experimental results of the BTR ($a_{train}=5, a_{pred}=15$) on the JFLEG corpus achieved the highest GLEU score for the whole corpus when $\lambda \leq 0.1$. This may be owing to the use of $a_{pred}=15$, which rendered a flatter probability than $a_{pred}=5$, as shown in Figure~\ref{fig:probability_distribution_of_ranking}. In addition, recognizing the fluency of a sentence by the BTR may be easier than recognizing the minimal edit of corrections.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 10
\begin{figure}[t]
\begin{center}
\includegraphics{31-1ia1f10.pdf}
\end{center}
\hangcaption{Average probability for each rank on the CoNLL-14 (left), BEA (middle), and JFLEG (right) test set. The top candidate sentences were generated by CTGEC}.
\label{fig:probability_distribution_of_ranking}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Difference Among Rerankers}
\label{sec:difference_among_rerankers}

To investigate the difference among \textcolor{black}{the CTR}, \textcolor{black}{ETR-P} ($\lambda=0.1$) w/o $a_{train}$, and the BTR ($\lambda=0.4$), we compared the precision and recall of the three rerankers in Table~\ref{table:main_result}.
In most cases, \textcolor{black}{the CTR} tended to improve the precision but lower the recall from \textcolor{black}{CTGEC}.
The improvements induced by \textcolor{black}{ETR-P} from \textcolor{black}{CTGEC} for both precision and recall were limited.
Meanwhile, the BTR improved both the precision and recall from the original ranking. Because \textcolor{black}{CTGEC} already achieved a relatively high precision and low recall, there was more room to improve recall, which was demonstrated by the BTR. Figure~\ref{fig:cross_entropy} shows that both \textcolor{black}{CTGEC} and \textcolor{black}{the CTR} exhibited a relatively high cross-entropy loss for tokens at the beginning positions and a low loss for tokens at the ending positions, although the loss of \textcolor{black}{the CTR} was the sum of two opposite decoding directions. 
This may be because the learning by the auto-regressive models for the latest token was over-fitting and for the global context was under-fitting, as reported by \citeA{qi-etal-2020-prophetnet}.
\textcolor{black}{ETR-P} exhibited a flatter loss with less sharp positions than \textcolor{black}{CTGEC} and \textcolor{black}{the CTR}.
Meanwhile, the BTR exhibited a flat loss, which was ideal for reranking candidate sentences with length normalization, as suggested by \citeA{salazar-etal-2020-masked}.
\textcolor{black}{The distinction between the BTR and CTR also indicates that the incorporation of bidirectional representation into the neural network structure aligns more closely with the characteristics of the cyclic directed graph, compared to the post-combination-based approach.}
Figure~\ref{fig:probability_distribution_of_ranking} shows the probability distribution of reranking.\footnote{\textcolor{black}{The results of the BTR ($a_{train} \ge 10$) and the ETR-P ($a_{train} \ge 10$) are presented in Appendix~\ref{sec:appendix_difference_among_rerankers}, as their trends align with those of the BTR ($a_{train}=5$) and the ETR-P ($a_{train}=5$), respectively.}}
When $a_{train} > 0$, the probability distribution of the BTR becomes skewed, which indicates that using Eq.~(\ref{eq:maxmize_likelyhood}) to minimize the unlikelihood increased the probability gap between the 1st-ranked candidate and the rest. Compared with the BTR, when $a_{train} > 0$, the probability distribution of \textcolor{black}{ETR-P} was as flat as $a_{train} = 0$, which suggests the effectiveness of the encoder-decoder structure in comparison with the encoder-only one when minimizing unlikelihood.
Appendix~\ref{sec:appendix_candidates} also presents the difference in training loss between \textcolor{black}{ETR-P} and the BTR.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 11
\begin{figure}[t]
\begin{center}
\includegraphics{31-1ia1f11.pdf}
\end{center}
\hangcaption{Cross-entropy loss of $y_j$ versus $j$. The loss was averaged over CoNLL-14's 149 tokenized utterances with length considered in interval $[18, 20]$ (including $<$eos$>$). } 
\label{fig:cross_entropy}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Table~\ref{table:example_of_reranked_outputs} provides examples of ranked outputs by \textcolor{black}{CTGEC}, \textcolor{black}{CTR}, \textcolor{black}{ETR-P} ($\lambda=0.1$) w/o $a_{train}$, and BTR ($\lambda=0.4$). 
The first block of output results demonstrates the difficulty of correcting spelling errors.
In this block, the BTR outputs the token ``insensitively'' with the correct spelling but a mismatched meaning, whereas other rerankers tended to retain the original token ``intesively'' with a spelling error. 
The examples in the second block showed that both the BTR and \textcolor{black}{the CTR} could correctly address verb tense errors.
The examples in the third block showed the methods utilized by various rerankers for handling count-mass noun errors. 
The examples in the last block showed that although the BTR recognized the missing determiner ``the'' for the word ``Disadvantage'', it still missed a that-clause sentence.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 14
\begin{table}[b]
\input{01table14.tex}%
\hangcaption{Examples of reranked outputs. The three candidate sentences were generated by \textcolor{black}{CTGEC}. \textcolor{black}{\textcolor{blue}{\underline{Underline}} indicates the range of corrections.} ``Candidate 1 (\textcolor{black}{CTGEC})'' denotes that \textcolor{black}{CTGEC} regards ``Candidate 1'' as having the most grammatical correction. 
Examples in the first two and last two blocks were extracted from the CoNLL-14 and JFLEG test corpus, respectively.}
\label{table:example_of_reranked_outputs}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 15
\begin{table}[b]
\input{01table15.tex}%
\hangcaption{Time cost (seconds) for inference over the whole corpus with five candidates generated by \textcolor{black}{CTGEC}.}
\label{table:inference_time_cost}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Table~\ref{table:inference_time_cost} displays the time cost for each model to estimate scores over the entire corpus with five candidates, using one NVIDIA A100 80GB GPU.
To estimate the time cost, we required all rerankers to compute one target sequence at a time. For \textcolor{black}{ETR-P} and the BTR, we rearranged the given target sequence by masking each token. 
These rearranged sequences were then placed into a mini-batch for parallel computation.
For \textcolor{black}{CTGEC}, given the source sentence, we used the mini-batch with a size of 5 to parallelly compute all beams. We only calculated the time for estimating probability and ignored the time for loading the model and dataset. 
As shown in the table, CTGEC required the most time among all rerankers, as it predicted tokens of the target sequence individually. \textcolor{black}{ETR-P} and the BTR required longer time than \textcolor{black}{ETR-C} and \textcolor{black}{CTR} owing to the target sequence rearrangement procedure. The BTR required 2 to 3 times as much as time as that required by \textcolor{black}{ETR-P} owing to the additional decoder structure.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}

We proposed a bidirectional transformer reranker (BTR) to rerank several top candidates generated by a pre-trained seq2seq model for GEC. For a fully pre-trained model, T5-base, the BTR achieved scores of 65.47 and 71.27 $F_{0.5}$ on the CoNLL-14 and BEA test sets. Our experimental results showed that the BTR on top of a T5-base with limited pre-training steps improved both the precision and recall for candidates from T5-base.
As using negative sampling for the BTR generated a skewed probability distribution for ranking, the introduction of a threshold $\lambda$ benefited the acceptance of the suggestion from the BTR.
\textcolor{black}{Using the BERT-style self-attention mechanism in the decoder of the BTR also aided in its close alignment with the cyclic directed graph.}
Furthermore, the BTR on top of T5-base could rerank candidates generated from T5-large and yielded better performance. This finding suggests the effectiveness of the BTR even in experiments with limited GPU resources. While the BTR in our experiments lacked sufficient pre-training, it should further improve the performance with full pre-training for reranking in future. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Limitations}

As mentioned in the previous section, up until now, no fully pre-trained seq2seq model with a BERT-style self-attention mechanism in the decoder has been proposed, while the vanilla seq2seq model tends to use a left-to-right or right-to-left unidirectional self-attention.
Therefore, utilizing our proposed BTR to rerank candidates from a pre-trained vanilla seq2seq model requires additional pre-training steps, which requires time and GPU resources.
\textcolor{black}{Similarly, the construction of the CTR involves additional pre-training steps owing to the unavailability of pre-trained right-to-left seq2seq models. However, the BTR still requires more training steps than a vanilla seq2seq model or the CTR. This is because the BTR masks and predicts only 15\% of the tokens in Eq.~(\ref{eq:maxmize_likelyhood}).}
%Because the BTR masks and predicts only 15\% of the tokens in Eq.~(\ref{eq:maxmize_likelyhood}), it requires more training steps than a vanilla seq2seq model \textcolor{red}{or the CTR}. 
In addition, during fine-tuning, \textcolor{black}{compared with the CTR that requires no additional hyperparameters $a_{train}$ and $\lambda$,} the BTR requires additional $a_{train}$ negative samples, which increases the fine-tuning period. Moreover, tuning $a_{train}$ will be inefficient if the training is slow. In other words, training an effective BTR requires considerably more time than training a vanilla seq2seq model or the CTR.
\textcolor{black}{Furthermore, The BTR requires 5 to 6 times more time than the CTR owing to the target sequence rearrangement procedure.}

As a reranker, the performance of the BTR depends on the quality of candidates. There is no room for improvement when using the BTR if no candidate is more grammatical than the original selection.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment

This paper is a revised version of \citeA{zhang2023bidirectional}, which has been accepted for publication at the Findings of the Association for Computational Linguistics: ACL 2023. In this version, we have made grammar revisions and merged the appendix with the main paper in Sections 3, 4, 5, and 6. \textcolor{black}{To aid the audience in better understanding this study, we have incorporated additional details to clarify our intent behind designing the BTR and adjusted the colors and styles used in our figures and tables.}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.7}
\bibliography{01refs}




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\appendix

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Hyperparameters}
\label{sec:appendix_hyperparameters}

Table~\ref{table:hyperparameters} lists the hyperparameter settings used for each model and Table~\ref{table:used_artifacts} lists the used artifacts.
The setting for \textcolor{black}{CTGEC} (large) was the same as \textcolor{black}{CTGEC}.
We followed the setting of \citeA{kaneko-etal-2019-tmu} to use a 0.0005 learning rate for the \textcolor{black}{ETR-C} reranker. We used a 0.0001 learning rate for the \textcolor{black}{ETR-P} reranker. For both \textcolor{black}{ETR-C} and \textcolor{black}{ETR-P}, the Adam optimizer, ``inverse square root'' learning rate schedule, and 1.2 epochs warm-up steps were used.
For other models based on a T5 structure, we used a 0.001 learning rate and Adafactor optimizer. The batch size was 1048576 tokens for all models.
We used the Fairseq \cite{ott-etal-2019-fairseq} and HuggingFace \cite{wolf-etal-2020-transformers} to reproduce all models and run the BTR.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 16
\begin{table}[t]
\input{01table16.tex}%
\caption{Used hyperparameters.}
\label{table:hyperparameters}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 17
\begin{table}[t]
\input{01table17.tex}%
\caption{Used artifacts.}
\label{table:used_artifacts}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 18
\begin{table}[t]
\input{01table18.tex}%
\hangcaption{Number of sentence pairs for cLang-8 dataset with candidates. All pairs of data that satisfy the length constraint of 128 are listed.}
\label{table:size_of_training_candidates}
%%%%\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\par\vspace{1\Cvs}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 19
%%%%\begin{table}[t]
\input{01table19.tex}%
\hangcaption{Results of tuning $a_{train}$ for BTR. $a_{pred}$ was set as 5. The highest $F_{0.5 }$ score on the CoNLL-13 corpus for each $a_{train}$ among different threshold is indicated in \textbf{bold}. The scores that were the same as those of the base model ($\lambda=1$) were ignored and \textcolor{gray}{greyed out}.}
\label{table:train_topa_result}
%%%%\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\par\vspace{1\Cvs}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 20
%%%%\begin{table}[t]
\input{01table20.tex}%
\hangcaption{Results of tuning $a_{train}$ for \textcolor{black}{ETR-P}. $a_{pred}$ was set as 5. The highest $F_{0.5}$ score on the CoNLL-13 corpus for each $a_{train}$ among different threshold is indicated in \textbf{bold}. The scores that were the same as those of the base model ($\lambda=1$) were ignored and \textcolor{gray}{greyed out}.}
\label{table:train_topa_result_RoBERTa}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Candidate and Threshold Tuning}
\label{sec:appendix_candidates}

Following \citeA{zhang-etal-2021-language}, we tuned $a$ to realize separate training and prediction on the validation dataset with candidates generated by \textcolor{black}{CTGEC}. Table~\ref{table:size_of_training_candidates} lists the size of training data with candidates generated by \textcolor{black}{CTGEC}.
When tuning $a_{train} \in \{0, 5, 10, 20\}$\footnote{Setting $a$ to 0 indicates training with only gold data.} for the BTR, $a_{pred}$ was fixed to 5. 
Because the BTR with $\lambda=0.4$ and $a_{train}=20$ achieved the highest score as shown in Table \ref{table:train_topa_result}, $a_{train}$ was fixed to 20, this BTR was also used to tune $a_{pred} \in \{5, 10, 15, 20\}$. 
When tuning $a_{train} \in \{0, 5, 10, 20\}$ for \textcolor{black}{ETR-P}, $a_{pred}$ was fixed to 5. The results in Tables \ref{table:train_topa_result} and \ref{table:train_topa_result_RoBERTa} present the different distributions of $F_{0.5}$ score between \textcolor{black}{ETR-P} and the BTR.
To investigate the reason, we compared the training loss and $F_{0.5}$ score of \textcolor{black}{ETR-P} with the BTR. Figure \ref{fig:diff_atrain} presents a comparison. In contrast to BTR, when using negative sampling ($a_{train} > 0$) for training \textcolor{black}{ETR-P}, the $F_{0.5}$ score on the CoNLL-13 corpus decreased with increase in the epoch. The training loss of \textcolor{black}{ETR-P} also reduced suddenly after finishing the first epoch. This result suggests that negative sampling in the GEC task for an encoder-only structure results in the wrong direction in learning representations from the concatenated source and target.
Therefore, we fixed $a_{train}$ to 0 for \textcolor{black}{ETR-P}. This \textcolor{black}{ETR-P} was also used to tune $a_{pred} \in \{5, 10, 15, 20\}$.
The results presented in Tables~\ref{table:eval_topa_result_BTR}, \ref{table:eval_topa_result_RoBERTa}, and \ref{table:eval_topa_result_R2L} show that when $a_{pred}$ was set to 5, \pagebreak
the BTR, \textcolor{black}{CTR}, \textcolor{black}{ETR-P}, and \textcolor{black}{ETR-C} attained their highest scores on the CoNLL-13 corpus. Thus, $a_{pred}$ was set as 5 in our experiments.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 12
\begin{figure}[t]
\begin{center}
\includegraphics{31-1ia1f12.pdf}
\end{center}
\hangcaption{Performances of BTR and \textcolor{black}{ETR-P} with various $a_{train}$ without $\lambda$ during fine-tuning. $a_{pred}$ was set as 5 with candidates from \textcolor{black}{CTGEC}. Both $F_{0.5}$ score and training loss were averaged over the four trials.}
\label{fig:diff_atrain}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 21
\begin{table}[p]
\input{01table21.tex}%
\hangcaption{Results of tuning $a_{pred}$ for BTR. The highest $F_{0.5}$ score on the CoNLL-13 corpus for each $a_{pred}$ among different threshold is indicated in \textbf{bold}. The scores that were same as those of the base model ($\lambda=1$) were ignored and \textcolor{gray}{greyed out}.}
\label{table:eval_topa_result_BTR}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 22
\begin{table}[p]
\input{01table22.tex}%
\hangcaption{Results of tuning $a_{pred}$ for \textcolor{black}{ETR-P}. The highest $F_{0.5}$ score on the CoNLL-13 corpus for each $a_{pred}$ among different threshold is indicated in \textbf{bold}. The scores that were same as those of the base model ($\lambda=1$) were ignored and \textcolor{gray}{greyed out}.}
\label{table:eval_topa_result_RoBERTa}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 23
\begin{table}[p]
\input{01table23.tex}%
\hangcaption{Results of tuning $a_{pred}$ for \textcolor{black}{CTR} and \textcolor{black}{ETR-C}. The highest $F_{0.5}$ score on the CoNLL-13 corpus for each reranker among different $a_{pred}$ is indicated in \textbf{bold}.}
\label{table:eval_topa_result_R2L}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 24
\begin{table}[p]
\input{01table24.tex}%
\hangcaption{Results of \textcolor{black}{ETR-P} and BTR on the CoNLL-13 corpus with candidates generated by \textcolor{black}{CTGEC} (large). The highest $F_{0.5}$ score for each reranker among different threshold is indicated in \textbf{bold}.
The scores that were the same as those of the base model ($\lambda=1$) were ignored and \textcolor{gray}{greyed out}.}
\label{table:valid_results_large}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Tables~\ref{table:train_topa_result} and \ref{table:eval_topa_result_BTR} present the performances of the BTR concerning $\lambda$ on the CoNLL-13 corpus with candidates generated by \textcolor{black}{CTGEC}.
Without using any candidate for training, the BTR ($\lambda=0$) achieved the highest $F_{0.5}$ score. When using 20 candidates for training, the BTR ($\lambda=0.4$) achieved the highest $F_{0.5}$ score of 50.22.
Table~\ref{table:valid_results_large} shows that the BTR ($a_{train}=20, \lambda=0.8$) achieved the highest $F_{0.5}$ score on the CoNLL-13 dataset with the candidates generated by \textcolor{black}{CTGEC}(large).
Thus, our tuned $\lambda$ for the BTR was set to 0.2 when $a_{train}=0$. When $a_{train}=20$, $\lambda$ was set to 0.4 and 0.8 for the candidates generated by \textcolor{black}{CTGEC} and \textcolor{black}{CTGEC} (large), 
respectively. Similarly, when $a_{train}=0$, our tuned $\lambda$ for \textcolor{black}{ETR-P} was set to 0.1 for the two versions of candidates.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 25
\begin{table}[b]
\vspace{-0.5\Cvs}
\input{01table25.tex}%
\hangcaption{Results of tuning $a_{train}$ and $a_{pred}$ for BTR on the BEA dev set. The highest $F_{0.5}$ score for each pair of $a_{train}$ and $a_{pred}$ among different threshold is indicated in \textbf{bold}. The scores that were the same as those of the base model ($\lambda=1$) were ignored and \textcolor{gray}{greyed out}.}
\label{table:tuning_a_for_BTR_on_BEA}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The BEA and JFLEG corpus also provide a dev set with 4384 and 754 sentences for validation, respectively. To determine the optimal $a_{train}$, $a_{pred}$, and $\lambda$ for the BTR listed in Table~\ref{table:train_topa_result} on these two datasets, we reevaluated the performances of the BTR on the corresponding dev sets. Tables~\ref{table:tuning_a_for_BTR_on_BEA} and \ref{table:tuning_a_for_BTR_on_JFLEG} show the results on the BEA and JFLEG dev sets, respectively. 
When using the BEA dev set, the highest $F_{0.5}$ score of 54.04 was achieved with $a_{train} = 10$, $a_{pred}=5$, and $\lambda=0.2$.
In case of the JFLEG dev set, the highest GLEU score of 54.46 was achieved with $a_{train} = 5$, $a_{pred}=15$, and $\lambda=0$.
These results demonstrate the differences in evaluating the minimal edit and fluency for grammar corrections. 
Considering the previous $a_{train}$, $a_{pred}$, and $\lambda$, we reevaluated the BTR on the BEA and JFLEG test sets. Table~\ref{table:tuned_a_lambda_for_BTR_onBEAJFLEG} lists the results. 
Tuning hyperparameters on the JFLEG dev set resulted in a higher GLEU score of 60.14 on the JFLEG test set, compared to the tuned hyperparameters on the CoNLL-13 set. However, tuning hyperparameters on the BEA dev set resulted in a lower $F_{0.5}$ score of 71.12 on the BEA test set, compared to the tuned hyperparameters on the CoNLL-13 set.
\pagebreak

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 26
\begin{table}[t]
\input{01table26.tex}%
\hangcaption{Results of tuning $a_{train}$ and $a_{pred}$ for BTR on the JFLEG dev set. The highest GLEU score for each pair of $a_{train}$ and $a_{pred}$ among different threshold is indicated in \textbf{bold}. The scores that were the same as those of the base model ($\lambda=1$) were ignored and \textcolor{gray}{greyed out}.}
\label{table:tuning_a_for_BTR_on_JFLEG}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 27
\begin{table}[t]
\input{01table27.tex}%
\hangcaption{Results for BTR on the BEA and JFLEG test sets with tuned hyperparameters. \textbf{Bold} scores indicate the highest $F_{0.5}$ and GLEU.}
\label{table:tuned_a_lambda_for_BTR_onBEAJFLEG}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{\textcolor{black}{Role of $\lambda$ on Test Set}}
\label{sec:appendix_role_of_lambda}

\textcolor{black}{Figures~\ref{fig:all_versus_lambda_conll_test}, \ref{fig:all_versus_lambda_bea_test}, and \ref{fig:all_versus_lambda_jfleg_test} show the performance of the BTR on the CoNLL-14, BEA, and JFLEG test corpus, respectively. Similar to the trends observed in Figures~\ref{fig:all_versus_lambda_conll}, \ref{fig:all_versus_lambda_bea}, and \ref{fig:all_versus_lambda_jfleg}, increase in $\lambda$ decreased the acceptance rate, while enhancing the precision and recall for the \emph{Accept} suggestions. 
In addition, the precision and recall over the entire corpus exhibited an initial increase followed by a decrease. Notably, the BTR consistently achieved the highest GLEU score for the entire corpus with a small $\lambda \leq 0.1$ on the JFLEG test corpus.}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 13
\begin{figure}[p]
\begin{center}
\includegraphics{31-1ia1f13.pdf}
\end{center}
\hangcaption{Precision and recall of BTR ($a_{train}=20, a_{pred}=5$) with respect to different $\lambda$ on the CoNLL-14 set.}
\label{fig:all_versus_lambda_conll_test}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 14
\begin{figure}[p]
\begin{center}
\includegraphics{31-1ia1f14.pdf}
\end{center}
\hangcaption{Precision and recall of BTR ($a_{train}=10, a_{pred}=5$) with respect to different $\lambda$ on the BEA test set.}
\label{fig:all_versus_lambda_bea_test}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 15
\begin{figure}[p]
\begin{center}
\includegraphics{31-1ia1f15.pdf}
\end{center}
\caption{GLEU of BTR ($a_{train}=5, a_{pred}=15$) with respect to different $\lambda$ on the JFLEG test set.}
\label{fig:all_versus_lambda_jfleg_test}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 16
\begin{figure}[p]
\begin{center}
\includegraphics{31-1ia1f16.pdf}
\end{center}
\hangcaption{Average probability for each rank of BTR ($a_{train} \ge 10$) and ETR-P ($a_{train} \ge 10$) on the CoNLL-14 (left), BEA (middle), and JFLEG (right) test set. The top candidate sentences were generated by CTGEC}.
\label{fig:probability_distribution_of_ranking_others}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{\textcolor{black}{Difference Among BTR and ETR-P}}
\label{sec:appendix_difference_among_rerankers}

\textcolor{black}{Figure~\ref{fig:probability_distribution_of_ranking_others} shows the performance of the BTR ($a_{train} \ge 10$) and the ETR-P ($a_{train} \ge 10$) on the CoNLL-14, BEA, and JFLEG test corpus, respectively. Similar to the observed trends in Figure~\ref{fig:probability_distribution_of_ranking}, when $a_{train} \ge 10$, the probability distributions of the BTR exhibited a peaked pattern, whereas those of the ETR-P appeared more flattened.}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Ying Zhang}{%
Ying Zhang is currently a Ph.D. candidate at the Department of Information and Communications Engineering, Tokyo Institute of Technology. She received her B.S. in Management from Chongqing University in 2017 and her M.E. from the Tokyo Institute of Technology in 2019. Her current research interests include natural language processing and machine learning.
}

\bioauthor[:]{Hidetaka Kamigaito}{%
Hidetaka Kamigaito received his Ph.D. from the Tokyo Institute of Technology and is currently an associate professor at the Nara Institute of Science and Technology. His current research interests include natural language processing with an emphasis on document-level text processing and knowledge graph completion.
}

\bioauthor[:]{Manabu Okumura}{%
Manabu Okumura was born in 1962. He received his B.E., M.E., and Dr. Eng. from Tokyo Institute of Technology in 1984, 1986, and 1989, respectively. He worked as an assistant at the Department of Computer Science, Tokyo Institute of Technology, from 1989 to 1992, and an associate professor at the School of Information Science, Japan Advanced Institute of Science and Technology, from 1992 to 2000. He is currently a professor at the Institute of Innovative Research at Tokyo Institute of Technology, Japan. His current research interests include natural language processing, particularly text summarization, computer-assisted language learning, sentiment analysis, and text data mining.
}
\end{biography}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\biodate



\end{document}

