\documentclass[english]{jnlp_3.7}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{hangcaption_jnlp}
\usepackage{array}
\allowdisplaybreaks


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition

%%%%%    \documentclass[english,dvipdfmx]{jnlp_3.7}
%%%%%    \usepackage{jnlpbbl_2.2}
%%%%%\usepackage[dvipdfmx]{graphicx}
%%%%%\usepackage{amsmath}
%%%%%\usepackage{url}
%%%%%\renewcommand\UrlFont{\rmfamily}

%%%%%\usepackage{times}
%%%%%\usepackage{latexsym}
%%%%%\usepackage{makecell}
%%%%%\usepackage{subcaption}
\usepackage{multirow}
\usepackage[ruled, linesnumbered]{algorithm2e}
%%%%%\usepackage{graphicx}
%%%%%\usepackage{comment}
\usepackage{colortbl,array,xcolor}
\usepackage{xspace}
%%%%%\usepackage{xcolor}
%%%%%\usepackage{tikz}
%%%%%\usepackage{amsfonts}
%%%%%\usepackage{soul}

\newcommand\corpus{\texttt{CELA}\xspace}
\definecolor{Gray}{gray}{0.935}
%%%%%\newcolumntype{g}{>{\columncolor{Gray}}c}

\newcommand\hill{\texttt{Hill}\xspace}
\newcommand\jiang{\texttt{Jiang}\xspace}
\newcommand\panda{\texttt{Panda}\xspace}
\newcommand\random{\texttt{Randomized}\xspace}
\newcommand\cloth{\texttt{CLOTH}\xspace}

%%%%%\newcommand{\bgfa}[1]{\tikz[baseline=(X.base)]{\node(X)[rectangle, fill=cyan!60!blue!18, rounded corners, text height=.8ex,text depth=-0.5ex]{\textit{#1}};}}
%%%%%\newcommand{\bgfb}[1]{\tikz[baseline=(X.base)]{\node(X)[rectangle, fill=red!60!blue!18, rounded corners, text height=.8ex,text depth=-0.5ex]{\textit{#1}};}}
%%%%%\newcommand{\bgfc}[1]{\tikz[baseline=(X.base)]{\node(X)[rectangle, fill=green!60!blue!18, rounded corners, text height=.8ex,text depth=-0.5ex]{\textit{#1}};}}
%%%%%\newcommand{\bgfd}[1]{\tikz[baseline=(X.base)]{\node(X)[rectangle, fill=gray!60!blue!18, rounded corners, text height=.8ex,text depth=-0.5ex]{\textit{#1}};}}

\newcommand\blfootnote[1]{%
  \begingroup
  \renewcommand\thefootnote{}\footnote{#1}%
  \addtocounter{footnote}{-1}%
  \endgroup
}

\newcommand{\rel}{$\textsf{REL}$\xspace}
\newcommand{\nrel}{$\textsf{NREL}$\xspace}
\newcommand{\suf}{$\textsf{SUF}$\xspace}
\newcommand{\sufg}{$\textsf{SUF\_G}$\xspace}
\newcommand{\sufr}{$\textsf{SUF\_R}$\xspace}
\newcommand{\nsuf}{$\textsf{NSUF}$\xspace}

%%%%%\newcommand\journal[1]{\textcolor[rgb]{1,0,0}{#1}}

%%%%%\newcommand\revise[1]{\textcolor[rgb]{0,1,0}{#1}}

\makeatletter
\renewcommand{\algocf@capseparator}{~~}%
\makeatother

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{31}
\Number{2}
\Month{June}
\Year{2024}
%%%%\Issuetitle{「　　　　」特集号}

\received{2023}{8}{31}
\revised{2023}{11}{30}
%%%%%\rerevised{2006}{5}{6}
\accepted{2024}{1}{5}

\setcounter{page}{328}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Cloze Quality Estimation for Language Assessment}
\eauthor{Zizheng Zhang\affiref{TMU} \and Masato Mita\affiref{CyberAgent}\affiref{TMU} \and Mamoru Komachi\affiref{Hitotsubashi}\affiref{TMU}} 
\eabstract{%
Cloze tests play an essential role in language assessment and help language learners improve their skills.
In this paper, we propose a novel task called \textbf{C}loze \textbf{Q}uality \textbf{E}stimation (CQE) --- a task of evaluating whether a cloze test is of sufficient ``high-quality'' for language assessment based on two important factors: reliability and sufficiency.
We have taken the first step by creating a new dataset named \corpus for the CQE task, which includes English cloze tests and corresponding evaluations about their quality annotated by native English speakers, which includes 2,597 and 1,730 instances in aspects of reliability and sufficiency, respectively.
We have tested baseline evaluation methods on the dataset, showing methods that only focused on the options would not perform well in the challenging task, especially in the aspect of reliability detection. More features such as context of questions are expected to improve the detection performance.
}
\ekeywords{Cloze Test, Lanugage Assessment, Language Education, Dataset}

\headauthor{Zhang et al.}
\headtitle{Cloze Quality Estimation for Language Assessment}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\affilabel{TMU}{}{Tokyo Metropolitan University}
\affilabel{CyberAgent}{}{CyberAgent, Inc.}
\affilabel{Hitotsubashi}{}{Hitotsubashi University}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}
\label{intro}
\blfootnote{Part of this research was published in the Findings of The 17th Conference of the European Chapter
of the Association for Computational Linguistics (EACL2023). The \corpus dataset is available at \url{https://github.com/zz-zhang/cloze-quality-estimation}.}
The English cloze test \cite{taylor1953cloze} was originally developed to measure the readability of texts. However, in recent years, it has gained popularity as a second language testing device capable of assessing students' language proficiency in a practical and efficient way \cite{trace2020clozing,dhyaaldian2022psychometric} (\textbf{Passage} and \textbf{Questions} in Table \ref{tab:task-exp}).
Cloze tests measure multiple language abilities of examinees simultaneously, for example, grammatical knowledge  \cite{alderson1979cloze,rye1982cloze} and reading comprehension ability \cite{raymond1988close,klein1997c}.
Various types of English cloze tests are utilized in language assessment, according to their own advantages. For example, in the aspect of blank selection (word deletion), fixed-ratio deletion allows educators to design and maintain questions accessibly, and rational deletion improves the purposes of questions; in the aspect of answers, 
\pagebreak
blank-filling questions focus on measuring the spelling of vocabulary while multiple-choice questions tend to test the understanding of the context.
The most widespread format of the cloze test is the multiple-choice word-level cloze test, which consists of an incomplete passage with several blanks and a series of questions, where each question includes several options (four options in the usual setting), requiring examinees to fill the blanks by selecting words (or phrases) from options that make the passage coherent.

The significantly high price of creating cloze tests by experts has prompted automatic cloze generation methods \cite{goto2010automatic,sakaguchi-etal-2013-discriminative,hill2016automatic,panda-etal-2022-automatic}.
However, automatically generated cloze tests do not always contribute well to language assessment and suffer from low quality.
In language assessment, a reliable and sufficient 
cloze test shows a high ability to measure examinees' language level \cite{bachman1985performance}.
Cloze question creation has two steps, word deletion and distractor generation, with the latter greatly affecting the ability to measure language level.
As indicated by \citeauthor{xie-etal-2018-large} \citeyear{xie-etal-2018-large}, some cloze tests, particularly automatically generated cloze tests, are created coarsely and cause two fatal issues in language assessment: 
(1) these tests do not guarantee that the answer is not ambiguous, which means there is a risk that multiple options fit almost equally well into the blank;
(2) the test creation process considers less about which aspect of the language phenomenon is measured in the test; hence, such tests cannot measure the examinee's language level.
These two issues make the cloze test unsuitable for measuring examinees' language level. 
The first issue makes the test unreliable (e.g., Question 4 in Table \ref{tab:task-exp}), which means even if an examinee has enough language knowledge to answer the test, the test might report a wrong score to indicate that the examinee lacks such knowledge.
In other words, an unreliable cloze test cannot present the examinee's language level. The second issue makes the test insufficient (e.g., Question 3 in Table \ref{tab:task-exp}). An insufficient test cannot easily identify the aspect in which an examinee lags in terms of language knowledge, which indicates that educators canhardly identify the knowledge that the examinee has not acquired.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{table}[t]
\input{01table01.tex}
    \hangcaption{Example of cloze test and qualities for each question in \corpus. The input consists of a \textbf{Passage} with multiple blanks and a series of \textbf{Questions} (tuples of options). The expected output is \textbf{Evaluation} tuples to indicate whether the questions are reliable (\rel for reliable and \nrel for not reliable)
    and sufficient (\suf for sufficient and \nsuf for not sufficient) and what language ability is tested if sufficient ($\textsf{G}$ for grammatical knowledge and $\textsf{R}$ for reading comprehension ability). Underlined options are the correct answers, which fit passage perfectly.}
    \label{tab:task-exp}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%




In this paper, we tackle the issues of evaluating the appropriateness of cloze tests for language assessment focusing on distractors.
By following the test design principle \cite{association2011manual}, we define a task to evaluate cloze tests for language assessment considering two aspects: reliability and sufficiency, which is called \textbf{C}loze \textbf{Q}uality \textbf{E}stimation (CQE).
In CQE, each question in a cloze test is asked to be estimated, whether it is reliable and sufficient.
CQE provides a cloze as input (\textbf{Passage} and \textbf{Questions} in Table \ref{tab:task-exp}) and requires estimation of each question as output (\textbf{Evaluation} in Table \ref{tab:task-exp}).
We introduce a new test set called the \textbf{C}loze \textbf{E}stimation dataset for \textbf{L}anguage \textbf{A}ssessment (\corpus) which includes a variety of cloze tests and corresponding annotations. 
These cloze questions are specifically designed for junior high-school students in China.
We prepared diverse English cloze tests including expert-designed and rule-generated tests, and asked native English speakers to solve them and annotate the quality of each question in the two aspects of reliability and sufficiency. 

We also introduce a framework for tackling the CQE task as baseline: we designed 
CQE methods that evaluate cloze questions by analyzing their options. 
We tested the baseline methods with the \corpus and compared them against statistical baselines.
We found that detection of unreliable questions is challenging and that all our baseline methods were wary to label a question as unreliable.
The framework of our CQE methods contributed to the sufficiency evaluation, particularly when implemented by DNN-based approaches, which outperformed statistical baselines significantly and showed potential for improvement.

Finally, we also test large language models (LLMs) on the \corpus. 
LLMs have shown remarkable performance across diverse tasks in zero-shot scenarios \cite{kojima2022large}. 
To explore the potential of LLMs in solving the CQE task, we tested the performance of LLMs with different prompts on \corpus.


Thus, the main contributions of this work are as follows:
(1) we propose a new task of quality estimation of cloze tests (CQE) for language assessment;
(2) we create a new CQE dataset (\corpus) for English learners, including annotations for both expert-designed and automatically generated cloze tests;
(3) we propose the baseline methods for CQE considering the options of cloze questions and show they can be good baselines to estimate the quality of cloze tests;
(4) we test LLMs on \corpus with different prompts, the result indicates that prompt engineering affects LLMs' performance significantly.


The remainder of this paper is organized as follows. We introduce related work in Section \ref{sect:related}.
In Section \ref{sect:task}, we define the CQE task and propose the corresponding dataset \corpus.
We design baseline methods for tackling CQE in Section \ref{sect:methods}.
In Section \ref{sect:exp}, We conduct experiments to evaluate baseline methods.
Section \ref{sect:conclu} concludes the paper and envisages future work.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Work}
\label{sect:related}
Language educators are capable of creating cloze tests rationally; they select words to be blanked and design distractors by their experience in language education to improve reliability and sufficiency. 
CLOTH \cite{xie-etal-2018-large}, SCDE \cite{kong-etal-2020-scde}, and CEPOC \cite{mariano2022cepoc} are collections of human-created cloze tests, which are highly evaluated by experts in terms of measuring the English ability of examinees.
However, designing cloze tests by experts is costly and difficult to generalize.

Automatic cloze generation methods could decrease the cost of creating cloze tests. To avoid generating useless tests in language assessment, these methods focus on distractor generation, which affects the quality of tests significantly.
Previous works have conducted trials designing good rules or machine learning models.
\citeauthor{sakaguchi-etal-2013-discriminative} \citeyear{sakaguchi-etal-2013-discriminative} described a method of generating distractors for assessing an English as second language (ESL) learner's ability to distinguish semantic nuances between vocabulary words.
They could generate sufficient questions, but these questions are domain-limited, which is not easy to generalize to cloze tests for human language assessment.
The Children's Book Test (CBT) \cite{hill2016cbt} deletes named entities, common nouns, verbs, and prepositions and then designs distractors having the same part of speech (POS) as the deleted words to measure abilities in reading comprehension and vocabulary.
Because of the naive distractor creation, the CBT method has the risk of generating unreliable questions. 
\citeauthor{coniam1997preliminary} \citeyear{coniam1997preliminary},
\citeauthor{goto2010automatic} \citeyear{goto2010automatic},
\citeauthor{correia2012automatic} \citeyear{correia2012automatic},
\citeauthor{hill2016automatic} \citeyear{hill2016automatic},
\citeauthor{jiang-etal-2020-know} \citeyear{jiang-etal-2020-know}
explore cloze test generation with various features, such as $n$-gram frequency and POS tag, and attempt to select deleted words and create distractors by using discriminative models including conditional random fields and support vector machine.
Advanced distractor generation methods \cite{panda-etal-2022-automatic} that employ large pre-trained language models (LMs) can provide more sufficient distractors. These LMs produce better text representation that captures rich semantic information, and better text representation allows generation methods to produce more plausible distractors, which could measure language abilities better.
However, designing good rules or models to improve the quality of cloze tests is not that easy. 
Furthermore, these works claimed they could generate better distractors, but it is difficult to perform comparisons to previous work. All the works performed human evaluations using their own metrics.



Crowdsourcing has been used to evaluate the quality of cloze tests and explore factors that affect quality  \cite{skory-eskenazi-2010-predicting}; workers were required to fill appropriate words in an open cloze-style sentence (a sentence with a blank, but without providing options).
Answers from workers were used to calculate Cloze Easiness \cite{finn1977wordfrequency} to indicate whether the sentence is usable for the testing an examinee's vocabulary.
The Association of Language Testers in Europe provides a manual for developing a language test \cite{association2011manual}, which specifies that the statistics of a test's results reflect its reliability and sufficiency.
It asks various examinees to answer a test and analyzes the statistics of a question such as the accuracy and the answer distribution.
However, these evaluation methods require human resources or experts, which is time-consuming and sometimes difficult to obtain.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Cloze Quality Estimation}  
\label{sect:task}
Motivated by related work, we propose the task of evaluating the quality of the cloze test.
We introduce the definition of CQE task and \corpus, our new dataset designed specifically for the CQE task.
Figure \ref{fig:cela_process} shows the flow of creating \corpus. We collected expert-designed and automatically generated cloze tests and asked native English speakers to annotate whether these tests are reliable and sufficient.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{figure}[t]
\begin{center}
\includegraphics{31-2ia1f1.pdf}
\end{center}
%%%%%    \includegraphics[width=0.63\textwidth]{figures/cela_flow.pdf}
    \hangcaption{Flow of creating \corpus. Cloze generation methods use passages and blanks from expert-designed tests to eliminate the effect of the word deletion strategy.}
    \label{fig:cela_process}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Task Definition}
\label{sect:task_defin}
First, we define a CQE task as follows. 
In a CQE task, given an incomplete passage with blanks and a series of questions with tuples of options, a quality estimation model should predict the quality of the questions and return tuples to indicate whether these questions are reliable and sufficient for language assessment.
Here, we formalize a CQE task as two classification sub-tasks: \textit{reliability evaluation} and \textit{sufficiency evaluation}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Reliability.}
In the reliability evaluation, given a cloze passage with multiple questions (tuples of options), we perform a binary classification of whether the questions are reliable (\rel) or not (\nrel).
In terms of reliability, if a cloze question has more than one option that fits the context perfectly, there is no guarantee that the question can report a stable test score even when taken by the same examinee.
Thus, in the reliability evaluation, we define that if a cloze question has more than one correct answer, it is not reliable.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Sufficiency.}
In the sufficiency evaluation, we perform a three-class classification of either the question is sufficient for measuring grammatical knowledge (\sufg), sufficient for measuring reading comprehension ability (\sufr), or insufficient (\nsuf) \cite{aryadoust2020extensive}.
In terms of sufficiency, a sufficient question should require examinees to use their specific language ability to distinguish the correct answer option from distractors (e.g. a \sufg question only requires grammatical knowledge to eliminate three distractors from four options).
If a question measures more than one language ability, the question is considered to be too simple to measure the language ability of the examinee \cite{rankin1977sequence}.
Thus, in the sufficiency evaluation, we define that if a question requires the examinee's single language ability (grammar or reading comprehension) to distinguish the answer option, the question is sufficient, otherwise it is insufficient.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{\corpus data preparation}
We collected English cloze tests from Chinese senior-high-school examinations \cite{xie-etal-2018-large} called \cloth, which is expert-designed.
To explore whether automatically generated cloze questions are sufficient for language assessment, we also employed four automatically generated cloze tests using previous generation methods: \random, \hill, \jiang, and \panda.
All generated tests were based on the same cloze passages from expert-designed tests, that is, these five settings share the passages, blanks, and correct answers but have respective distractors in questions.

\random is generated using a random sampling method.
In this method, we built vocabulary from \cloth and randomly selected words from the vocabulary as distractor options. 

\hill is generated using the same method of the CBT dataset \cite{hill2016cbt}, which selects words that have the same POS tag with the answer from the vocabulary as distractors.

\jiang  employs the method of \citeauthor{jiang-etal-2020-know} \citeyear{jiang-etal-2020-know}, which also selects words from the vocabulary but considers more factors including POS tag, word frequency, and spelling similarity. Their method is designed for the Chinese cloze test, but we adapted it to the English test.

\panda employs the method of \citeauthor{panda-etal-2022-automatic} \citeyear{panda-etal-2022-automatic}, which uses round trip translation to paraphrase a passage and align the paraphrased passages with the original one. They use aligned words to the answer as distractor candidates and select a distractor from candidates considering the synonym and POS tag.

As a result, we collected and generated 150 cloze tests including 3,000 questions. The cloze tests are collected/generated in five ways, each accounting for one-fifth of the total. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{table}[t]
\input{01table02.tex}
    \hangcaption{Example of annotation. We used following instructions: ``Please select an option in the number of answers list to indicate whether there is more than one option that fits the passage perfectly; please select what kind of language ability the question measures in the measured ability list.}
    \label{tab:instruction}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{\corpus annotation}
We hired Amazon Mechanical Turkers to annotate the 3,000 questions.
To ensure annotation quality, we required annotators to have approval rates over 98\% and be native English speakers living in the United States.
We also added attention checks to avoid bots and irresponsible annotators.
Each question was annotated by three different annotators.
Table \ref{tab:instruction} shows examples of our annotation task.
As a reward, we paid each annotator \$1.5 for a test, which included 20 questions and took 5 to 7 minutes for completion.



We performed inter-annotator analysis on the annotations using Fleiss' kappa score \cite{fleiss1971measuring}.
Kappa scores were 0.67 and 0.45 for reliability (binary) and sufficiency (3-class), respectively.
Moderate kappa scores indicate that the annotation task was well-defined and the annotation result was trustable.
Furthermore, to improve the annotation quality, we discarded all disagreed annotations. 
The majority of annotations that were rejected on the grounds of reliability pertained to long-term reasoning questions. 
These questions necessitated the integration of information from multiple sentences, and without taking into account this information, the distractors appeared to be equally plausible. 
This led to a divergence of opinions among some annotators and ultimately resulted in the determination that these questions were unreliable.
The reasons for rejection in terms of sufficiency were more varied. 
    One pattern that emerged was the use of prepositions, where some annotators classified questions regarding preposition usage as {\sufr} instead of \sufg, despite our explicit instructions on this matter. 
We posit that this may have been due to the fact that certain questions involving prepositions necessitate contextual information in order to deduce the correct answer (e.g., prepositions of location), causing some annotators to consider them as reading comprehension questions.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{table}[t]
\input{01table03.tex}
    \hangcaption{Statistics of the processed data. Because reliability is easier to annotate, it has higher agreement, and more annotations are retained than sufficiency.}
    \label{tab:processed_data}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



The processed data statistics are shown in Table \ref{tab:processed_data}.
Because most blanks in \cloth are content words and corresponding questions are designed to measure reading comprehension ability, there are few questions that measure grammatical knowledge.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{\corpus analysis}
We observed that the five types of cloze tests have various qualities. 
Figure \ref{fig:cela_types} shows the quality statistic in \corpus according to generation methods.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{figure}[t]
%%%%%    \begin{subfigure}[b]{0.495\textwidth}
%%%%%    \includegraphics[width=\textwidth]{figures/cela_rel2.pdf}
%%%%%    \caption{Reliability}
%%%%%    \label{fig:cela_rel}
%%%%%    \end{subfigure}
%%%%%    \begin{subfigure}[b]{0.495\textwidth}
%%%%%    \includegraphics[width=\textwidth]{figures/cela_val2.pdf}
%%%%%    \caption{Sufficiency}
%%%%%    \label{fig:cela_val}
%%%%%    \end{subfigure}
\begin{center}
\includegraphics{31-2ia1f2.pdf}
\end{center}
    \hangcaption{Quality statistics of cloze tests in \corpus. \rel, \sufr, and \sufg denote high-quality. \nrel and \nsuf denote low-quality.}
    \label{fig:cela_types}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


In reliability, \jiang is the most reliable and only includes 3.9\% of unreliable questions, and \panda has 22.1\%, which is the most unreliable.
Surprisingly, \cloth and \panda, which are expert-designed and generated by an advanced generation method, respectively, are not as reliable as the others.
We conjecture that these two types of tests tend to produce more plausible distractors that break only little coherence of the context. Plausible distractors are good at measuring learners' language ability but have a higher risk of making the question unreliable. 
In particular, the \panda system utilizes round-trip translation and alignment to generate distractor candidates, which limits the scope of possible candidates and tends to produce more credible options compared to those generated by other systems. 
Furthermore, the \panda system does not impose strict limitations on eliminating distractors that are also suitable for the blank, which increases the likelihood of generating unreliable questions. 
On the other hand, the \random system selects distractors from the vocabulary without any constraints, which reduces the chance of selecting distractors that are also appropriate for the blank.


In sufficiency, meeting our conjecture, there are fewer insufficient questions in \cloth and \panda, which means these two test types are better at measuring language ability than others.
For automatic distractor generation methods, \panda has the strictest restrictions on distractor selection and produces the fewest insufficient questions. 
\jiang has more filters for eliminating distractor candidates than \hill and could generate more sufficient questions. 
\random does not have any restrictions and is difficult to produce sufficient questions for language assessment.

We provide an example of the \corpus dataset in Table \ref{tab:task-exp}.
In the \corpus dataset, each instance includes an incomplete passage with blanks and corresponding sets of options as input (questions). 
In a question, at least one option can be filled into the corresponding blank to make the passage coherent both grammatically and semantically. 
The label for each question is a tuple that denotes whether the question is reliable and sufficient, and if the question is sufficient the tuple also indicates which aspect of language ability the question measures.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{CQE Method}
\label{sect:methods}
We propose two methods to tackle the CQE task, which analyze all options of cloze questions as baseline methods for the CQE task.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Intuition}
\label{sect:intui}
We designed a CQE method considering how options in the question affect reliability and sufficiency. 
We followed the definition in Subsection \ref{sect:task_defin} and considered that the reliability and sufficiency of a question is decided by its options.
Thus, to tackle two sub-tasks in CQE, we need to inspect each option in terms of (1) whether it can be regarded as the sole answer to the question, and (2) what language ability it measures.

For the former (reliability), we consider that if an option breaks neither grammatical nor semantic coherence of the context, it fits the context perfectly and can be regarded as an answer option. 
For the latter (sufficiency), if a distractor option only breaks grammatical (or semantic) coherence, examinees will use grammatical knowledge (or reading comprehension ability) to eliminate it, and in these cases, we say the distractor option is a grammatical (or reading) option; if a distractor option breaks both coherence, because it is too simple to measure one's ability, we say it is a purposeless option.

For example, given a context: \textit{I remember sitting in that dark hall listening to Mr. Zigler \_\_\_\_  everyone's spirits up to the ceiling.} and options: \textit{[raise, rise, educate, disappointed]}, the option \textit{raise} does not break neither grammatical nor semantic coherence, so it is an answer option; the option \textit{rise} breaks the grammatical coherence because the blank requires a transitive verb, so it is a grammatical option; the option \textit{educate} obeys the grammatical rule but does not fit context semantically, so it is a reading option; the option \textit{disappointed} is a purposeless option because it breaks both grammatical and semantic coherence of context.

Based on this intuition, we implement two functions, $BreakGrammar(\cdot)$ and {\linebreak}$BreakSemantics(\cdot)$, to judge whether an option breaks grammatical or semantic coherence.
The detailed description of the overall framework is shown in Algorithm \ref{algo:framework}.
To realize these two functions, we designed two different approaches: a rule-based approach and a DNN-based approach.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{algorithm}
\input{01algorithm01.tex}
\caption{Framework of CQE method}
\label{algo:framework}
\end{algorithm}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Rule-based approach}
\label{sect:rulebased}
The rule-based approach is straightforward. It compares options with the answer to a question. 
The answer to a question can fit the context perfectly and does not break either grammatical or semantic coherence. 
Thus, we consider that if an option has the same grammatical/semantical feature as the answer, it does not break corresponding coherence either. 
In this case, functions $BreakGrammar(\cdot)$ and $BreakSemantics(\cdot)$ 
require one more parameter $\mathit{answer}$.


Given an answer option $\mathit{answer}$ and an option $\mathit{opt}$, we fill $\mathit{answer}$ and $\mathit{opt}$ into context and obtain POS tags for them. If $\mathit{opt}$ has the same POS tag as $\mathit{answer}$, we consider that it does not break grammatical coherence, otherwise it breaks grammatical coherence. 
For the implementation, we employed POS tagger in the Stanza library.\footnote{\url{https://github.com/stanfordnlp/stanza}}

Similarly, we use a synonym dictionary to judge if the option breaks grammatical coherence.
If $\mathit{opt}$ is a synonym of $\mathit{answer}$, $\mathit{opt}$ does not break the semantic coherence, otherwise it breaks semantic coherence. 



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{DNN-based approach}
\label{sect:dnnbased}


We also designed a DNN-based approach to implement these two functions.
By using pretrained DNN models, we can plug in both grammatical and semantic knowledge into the CQE model.
Unlike the rule-based approach, the DNN-based approach does not use $\mathit{answer}$ but $\mathit{opt}$ information for CQE.



We employ an English grammatical error corrector that can detect both grammatical and semantic errors and output the error types.
We fill each option into context as input of the corrector and check the output.
If the output indicates that there is no grammatical/semantic error, we regard that the option does not break grammatical/semantic coherence; otherwise, we think it breaks such coherence. 
We need to distinguish grammatical and semantic errors which affect the output of $BreakGrammar(\cdot)$ or $BreakSemantics(\cdot)$. We design such a filter based on error types.
To recognize the error type, we use the output tag of an error annotation toolkit. 



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experiment}
\label{sect:exp}
We conduct experiments to determine whether CQE methods (\S\ref{sect:methods}) can be a good baseline to estimate the quality of cloze tests, by comparing them with statistical and LLM baseline methods (\S\ref{sect:baseline}, \S\ref{sect:llm}).



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Configurations}
\label{sect:setting}
To implement CQE method with a rule-based approach, we built an English synonym dictionary.\footnote{collected from \url{https://www.thesaurus.com/}}
Considering that the word inflection or tense do not affect the meaning, we lemmatized both $\mathit{answer}$ and $\mathit{opt}$ into their basic form to judge if they were synonyms.
The word lemmatization was implemented by employing the NLTK library\footnote{\url{https://www.nltk.org/}} and using the lemmatizer based on WordNet \cite{miller1998wordnet}. 
We also employed POS tagger in the Stanza library to assign POS tags to $\mathit{answer}$ and $\mathit{opt}$.

As for a DNN-based approach, we employed GECToR \cite{omelianchuk-etal-2020-gector}, a grammatical error corrector, that provided trained parameters and achieved a considerable performance on both CoNLL-2014 and BEA-2019 shared task \cite{conll2014,bryant-etal-2019-bea}. We used GECToR which was implemented by RoBERTa \cite{roberta}.
We fed original and corrected sentences into the ERRor ANnotation Toolkit (ERRANT)\footnote{\url{https://github.com/chrisjbryant/errant}} to obtain ERRANT tags.
If the detected error's ERRANT tag is one of \textbf{ADJ}, \textbf{ADV}, \textbf{NOUN}, and \textbf{VERB}, we considered the error to be a semantic one and not a grammatical one. 
Furthermore, we observed that the tag \textbf{OTHER} might contain both grammatical and semantic errors; therefore, we set two configurations for errors with tag \textbf{OTHER} as either grammatical or semantic errors.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Statistical baselines}
\label{sect:baseline}
We employed the following \textit{random} baseline and \textit{majority prediction} baseline to show how well statistical methods could perform on the \corpus.
Statistical baselines can also be regarded as weak baselines.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Random baseline}
The random baseline predicts random class in reliability and sufficiency classification. 
We chose the output class from the uniform distribution.

\paragraph{Majority prediction baseline}
The majority prediction baseline predicts the majority class in each classification sub-task.
According to our \corpus dataset, it always predicts \rel and \sufr in the sub-task of reliability and sufficiency classification, respectively.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{LLM baselines}
\label{sect:llm}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{table}[t]
\begin{center}
\input{01table04.tex}
\end{center}
	\caption{Zero-shot prompts for LLMs.}
	\label{tab:prompt}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



We employed GPT3.5-turbo\footnote{\url{https://platform.openai.com/docs/guides/gpt}} to explore the potential of LLMs on \corpus.
We designed zero-shot prompts for reliability and sufficiency, respectively.
The prompts we used are shown in Table \ref{tab:prompt}.
To invesgate whether LLMs could perform better with more information, we also designed few-shot prompts. 
Specifically, we add annotation examples shown in Table \ref{tab:instruction} in zero-shot prompts.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Meta-evaluation metrics} % -> 6.2
To demonstrate the efficiency of CQE methods in estimating the quality of cloze tests, we provide baseline meta-evaluation metrics for the CQE task.
Specifically, in the reliability evaluation, we used $F_1$, precision, and recall score. 
Because unreliable cloze tests are harmful to language assessment, we must focus on how well CQE models can recognize unreliable tests; thus we set \nrel as the positive label.
For the sufficiency evaluation, we used the micro-averaged and macro-averaged $F_1$ score.
To indicate how well models perform in each class, we also split the overall $F_1$ score into three parts: $F_1$ for \sufr, \sufg, and \nsuf.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Result}
\label{sect:result}
\begin{table*}[t]
\begin{center}
\input{01table05.tex}
\end{center}
    \hangcaption{Performance of CQE baseline methods on the \corpus dataset. $r.F_1$, $g.F_1$, and $n.F_1$ represent binary $F_1$ score for \sufr, \sufg, and \nsuf questions, respectively. \textbf{Bold} and \underline{underline} indicate the best and second-best result, respectively. $\bar O$ and $O$ indicate we regard errors from GECToR with tag \textbf{OTHER} as grammatical and semantic errors, respectively.}
    \label{tab:result}
\end{table*}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



The performance of the baselines on the \corpus dataset is presented in Table \ref{tab:result}. 
For statistical baselines, because of imbalanced data distribution, the majority prediction baseline was not able to detect the \nrel questions. 
Both baselines of random and majority prediction did not perform well on reliability compared with sufficiency. 
Moreover, unreliable question detection is important to language assessment. In future work, improving the performance on reliability should be considered preferentially.

The CQE baseline implemented by the rule-based approach performed worse than random baselines on some metrics. 
Although it achieved a moderate recall value, the precision was nearly zero, which denotes it tends to assign \rel to all questions. 
On the sufficiency performance, it outperformed statistical baselines on some metrics, but it is still insufficient for evaluating the quality of cloze tests.
One reason is that rules using the POS tag and synonym list are so na\"{i}ve that they only consider partial cases of the option type. 
For example, given a context \textit{This music made everyone want to \_\_\_\_. It was an early form of jazz.} and options \textit{[\underline{dance}, sing, laugh, ...]}, though options \textit{sing} and \textit{laugh} are not the synonyms of the answer \textit{dance}, they also fit the context semantically and should have not been classified into the reading option.

In most cases, the CQE method using the DNN-based approach outperformed statistical baselines. 
The DNN models utilized in this paper were straightforward and rudimentary, and there is potential for further improvement to make them more suitable for widespread use.
Regarding reliability, errors with \textbf{OTHER} as grammatical or semantic errors have little effect on the performance. 
In terms of sufficiency, when we regard \textbf{OTHER} errors as semantic errors, the micro $F_1$ value increased because the model could predict more \nsuf questions correctly, which accounted for a significant proportion in \corpus.

Except for hyperparameters, the mis-prediction caused by the underlying DNN models also leads to errors. GECToR did not perform well on long-term reasoning; thus, it was not able to detect some semantical errors. For example, given a context \textit{I was \_\_\_\_ of flying, ... In order to get rid of my fear I decided to try a helicopter ride}, when filling word \textit{proud} into the blank, we expect GECToR to correct the sentence with some words similar to \textit{afraid}, but GECToR did not report any error.

LLMs did not perform well in the case of zero-shot prompting, which means LLMs are not able to understand the task well when providing only instructions in prompts.
Once providing several examples, LLMs could perform better, especially in $recall$ of reliability and $r.F_1$ of sufficiency.
As a new task, CQE requires more detailed instructions in prompts to help the model understand.




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion and Discussion}
\label{sect:conclu}
We proposed a novel task for evaluating cloze questions for human language assessment (CQE), which involved two important factors that affect the quality of cloze questions, reliability and sufficiency, and also provided the \corpus dataset.
In addition, we explored automated CQE methods that can estimate the quality of cloze tests, by designing a method that focuses on options in a question.
Our experimental results on the \corpus dataset showed that imbalanced data bring challenges.

However, there are several limitations to this study. The first limitation of this study is the coverage. In this study, the CQE task is defined to evaluate cloze tests that are generated by distractor generation methods. 
Cloze tests in this study are all based on expert-designed blanks. However, word deletion methods, which decide which word to be blanked, affect the quality of cloze tests, too.
Investigation of how blanks influence the quality of cloze tests is necessary.

The second limitation of this study is the scalability of the annotation.
In this study, the annotation of question quality is done by experts, which makes creating a large-scale dataset challenging.
This limitation could be mitigated by alternative choice of target data, i.e., there is room to replace native speakers with non-native speakers by selecting target data that do not require English knowledge at high-level proficiency  (e.g., CEFR-A).

Finally, the CQE task and corresponding corpus are designed specifically for the English language. However, we are interested in exploring the possibility of adapting the task and dataset to other languages. The principles of test design, such as reliability and sufficiency, apply to other languages as well, but the specific details may vary based on the language. For example, questions for a hieroglyph-based language may require learners to identify glyphs, which must be taken into consideration when defining reliability and sufficiency. Adapting the CQE task to a new target language and creating a corresponding dataset requires a publicly available cloze question dataset or effective cloze question generation techniques in the target language, as well as experts in the language to evaluate question quality. In the future, we hope to develop an automatic adaptation method to transfer our task and dataset to multiple languages.

Therefore, in future work, we would like to investigate
more factors that affect the quality of cloze questions and expand the \corpus dataset, such as the blank selection. 
For example, selecting a ``easy'' word to be blanked would make the question insufficient in language assessment --- the question is too easy to distinguish learners with different language levels.
To tackle this issue, we would like to implement blank selection strategies and employ experts to annotate whether the blanks are appropriate for language assessment --- an inappropriate blank would make the question low-quality.

We would also like to investigate more factors that affect the quality of cloze questions and expand the \corpus dataset. 
For example, given the context \textit{The sun also \_\_\_\_.} and two different sets of options \textit{[raises, rises, lifts, elevates]} as well as \textit{[raises, lives, runs, lefts]}, although these two sets of options both measure reading comprehension ability, answering the question with the former set of options is more difficult than the latter and requires a higher language level. 

Improving the performance of baseline methods is also valuable for future work.
For DNN models, optimizing implementation of $BreakGrammar(\cdot)$ and $BreakSemantics(\cdot)$ can improve the evaluation performance. 
For example, designing more detailed filters in the grammatical error corrector to filter out potential semantic errors or using fine-designed data to train different language models for grammatical and semantic error detection separately may boost the performance of DNN models. 
For LLMs, we believe that designing prompts that could describe the CQE task well and include abundant examples would improve the LLMs' performance significantly.


Finally, while cloze tests and the practice of dividing language into detailed components remain prevalent in global language education, it is undeniable that these approaches reflect older educational paradigms. 
In this context, exploring the potential of NLP to rejuvenate and align language education with current educational theories presents a compelling avenue for future research. 
We envisage a future where NLP not only complements traditional methods but also synergizes with innovative educational frameworks to facilitate more holistic and nuanced language learning experiences. 

We hope that the task and our resource will encourage further exploration from both computational linguistics and language education.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment
This work was supported by JST, the establishment of university fellowships towards the creation of science technology innovation, Grant Number JPMJFS2139.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    \bibliographystyle{jnlpbbl_1.7}
\bibliography{01refs}




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\begingroup
\addtolength{\baselineskip}{-1.5pt}

\bioauthor[:]{Zizheng Zhang}{%
Zizheng Zhang received his M. Eng. from Kyushu University, Japan. He received his Ph.D. degree from Tokyo Metropolitan University, Japan. His research interests include natural language processing, particularly dataset creation and few-shot/zero-shot text classification/generation.
}

\bioauthor[:]{Masato Mita}{
He graduated from Department of Management and Information Science, Prefectural University of Hiroshima in 2014.
He graduated from Graduate School of Information Science and Technology, Nara Institute of Science and Technology in 2016.
In the same year, he joined Microsoft Japan.
He joined RIKEN Center for Advanced Intelligence Project (AIP) in 2018.
He recived Ph.D. (Information Science) from Graduate School of Information Science, Tohoku University in 2021.
He joined CyberAgent in 2022 as a research scientist.
}
\bioauthor[:]{Mamoru Komachi}{
Mamoru Komachi is a Professor at Hitotsubashi University.
He received his M.Eng. and Ph.D. degrees from Nara
Institute of Science and Technology (NAIST) in 2007 and 2010. He worked at NAIST and Tokyo Metropolitan University before joining Hitotsubashi University. His research
interests include syntax/semantics/discourse and educational applications of natural language processing.
}


\endgroup


\end{biography}

\biodate   %%%% Following settings will be edited by editors.


\end{document}

