\documentclass[english]{jnlp_3.8}
\usepackage{jnlpbbl_2.3}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage[hyphens]{url}
\usepackage{array}
\usepackage{hangcaption_jnlp}
\allowdisplaybreaks


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
%%%%\usepackage[dvipdfmx,hidelinks]{hyperref}
    \usepackage{prettyref}
    \let\autoref=\prettyref
\newrefformat{tab}{Table \ref{#1}}
\newrefformat{fig}{Figure \ref{#1}}
%%%%\renewcommand{\sectionautorefname}{Section}
%%%%\renewcommand{\subsectionautorefname}{Section}
%%%%\renewcommand{\subsubsectionautorefname}{Section}
%%%%\newcommand{\appref}[1]{\hyperref[#1]{Appendix~\ref*{#1}}}
    \newcommand{\appref}[1]{{Appendix \ref{#1}}}
\usepackage{ifthen}
\usepackage{xstring} % \IfBeginWith
\usepackage{xparse}
\NewDocumentCommand{\Rev}{s o m m}{%
    #4%
}
\usepackage{pxrubrica}  % /ruby
\usepackage{fontawesome5}
%%%%\usepackage{bold-extra} % enables bold with small caps: \textbf{\textsc{}}
\usepackage{booktabs}
\usepackage{multirow}
\usepackage{bigdelim} % \rdelim
\usepackage{arydshln}
\usepackage{pifont} % \ding
\usepackage{mathabx} % \dlsh
\newcommand{\ttnl}{$\dlsh$\\}
\usepackage{enumitem}
\usepackage{siunitx}
\usepackage{upgreek}
\sisetup{detect-weight=true} % can use \bfseries
%%%%\NewDocumentCommand{\UlNum}{m}{%
%%%%    \IfBeginWith{#1}{-}{%
%%%%        \underline{$#1$}%
%%%%    }{%
%%%%        \phantom{$-$}\underline{$#1$}%
%%%%    }%
%%%%}
\NewDocumentCommand{\UlNum}{m}{%
    \IfBeginWith{#1}{-}{%
        \underline{$#1$}%
    }{%
        \phantom{$-$}\underline{$#1$}%
    }%
}
\usepackage{umoline}    % \Midline
\newcommand{\shortul}[1]{\kern 1.5pt\underline{\kern -1.5pt #1\kern -1.5pt}\kern 1.5pt}
\newcolumntype{C}[1]{>{\centering\let\newline\\\arraybackslash\hspace{0pt}}p{#1}}
\newcolumntype{L}[1]{>{\raggedright\let\newline\\\arraybackslash\hspace{0pt}}p{#1}}
\usepackage{xspace}
\newcommand{\CBD}{\mbox{CBDJLS}\xspace}
\newcommand{\SNOW}{SNOW E4\xspace}
\newcommand{\JaLeCoN}{JaLeCoN\xspace}
\newcommand{\MLSJ}{MultiLS-Japanese\xspace}
%%%%\newcommand{\GScale}{\textsc{G-scale}\xspace}
\newcommand{\fakeGScale}{G-\resizebox{!}{1.2ex}{SCALE}\xspace} % works with italics
\newcounter{tblfn}
\newcommand{\tblfnsym}{\alph}
\newcommand{\tblfn}[1]{\setcounter{tblfn}{#1}\textsuperscript{\tblfnsym{tblfn}}}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{32}
\Number{4}
\Month{December}
\Year{2025}
%%%%\Issuetitle{Special Issue: XXXXXXXX}

\received{2025}{4}{1}
\revised{2025}{6}{28}
\accepted{2025}{8}{5}

\setcounter{page}{1129}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{A Japanese Dataset and Efficient Multilingual LLM-Based Methods for Lexical Simplification and Lexical Complexity Prediction}
\eauthor{Adam Nohejl\affiref{NAIST} \and Akio Hayakawa\affiref{UPF} \and Yusuke Ide\affiref{NAIST} \and Taro Watanabe\affiref{NAIST}} 
\eabstract{%
Lexical simplification (LS) is the task of making text easier to understand by replacing complex words with simpler equivalents. LS involves the subtask of lexical complexity prediction (LCP). We present MultiLS-Japanese, the first unified LS and LCP dataset targeting non-native Japanese speakers, and one of the ten language-specific MultiLS datasets. We propose methods for LS and LCP based on large language models (LLMs) that outperform existing LLM-based methods on 7 and 8 of the 10 MultiLS languages, respectively, while using only a fraction of their computational cost. Our methods rely on a single prompt across languages and introduce a novel calibrated token-probability scoring technique, \textsc{G-Scale}, for LCP. Our ablations confirmed the benefits of \textsc{G-Scale} and of concrete wording in the LLM prompt. We made the MultiLS-Japanese dataset available online under a CC-BY-SA license, including detailed metadata.
}
\ekeywords{Automated Text Simplification, Lexical Simplification, Lexical Complexity \linebreak Prediction, Large Language Models}

\headauthor{Nohejl et al.}
\headtitle{A Japanese Dataset and Efficient Multilingual LLM-Based Methods for LS \& LCP}

\affilabel{NAIST}{}{Nara Institute of Science and Technology}
\affilabel{UPF}{}{Universitat Pompeu Fabra}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

\renewcommand{\thefootnote}{}
\footnote[0]{Sections \ref{sec:dataset-overview} and \ref{sec:dataset-multils} of this article partially overlap in content with previous publications \cite[Sec.~4.1.6 and Sec.~2, respectively]{shardlow-etal-2024-bea,nohejl-etal-2024-difficult}; part of the \MLSJ dataset was briefly described in these publications. As part of this work, the complete dataset is made available at \url{https://huggingface.co/datasets/naist-nlp/multils-japanese}}
\renewcommand{\thefootnote}{\arabic{footnote}}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}

Lexical simplification (LS) is the task of making text easier to understand by replacing complex words with simpler equivalents. Therefore, it is a specific type of automatic text simplification (ATS) that operates at the word level without changing the overall sentence or text structure.

An LS system can serve as a reading assistance tool for end users, a tool for human editors, or a component of a larger text simplification system. It has been studied in the context of various target populations, such as non-native speakers \cite[\emph{inter alia}]{hading-etal-2016-japanese,paetzold-specia-2016-understanding,lee-yeung-2018-personalizing}, or people with dyslexia, aphasia, or low literacy levels \cite{paetzold_specia_2017_survey}.

The LS process is often broken down into two subtasks: (1) lexical complexity prediction (LCP) and (2) LS of the identified complex words, which can be further conceptualized as a multi-step pipeline \cite{paetzold_specia_2017_survey}. LCP can also be used for other purposes, such as readability assessment.

LS datasets are scarce for languages other than English. For Japanese, only two LS datasets with an unspecified target audience and one LCP dataset for non-native speakers are available. Because each target audience has very different needs, we constructed a dataset targeting non-native Japanese speakers, \MLSJ, which provides gold labels for both LS and LCP. We carefully designed the dataset considering the target audience and the specifics of the Japanese language: we sampled sentences from varied genres read by non-native speakers, included target words from categories that pose difficulties to learners (e.g., compound verbs), and substituted the appropriate units given the Japanese morphology. For lexical complexity annotation, we excluded Chinese and Korean L1 speakers, whose native languages share a large part of their vocabulary with Japanese. The LS annotators were native speakers with experience teaching non-native Japanese learners.

\MLSJ is part of a wider family of ten language-specific MultiLS datasets. We therefore designed and evaluated LS and LCP methods that are applicable to any of these languages. Similar to current state-of-the-art methods, our methods are based on large language models (LLMs); in contrast, our methods use a single LLM inference per data instance and a novel calibrated token-probability-based scoring technique, \textsc{G-Scale}, for LCP. Thus, we reduced the computational cost by an order of magnitude compared to other LLM-based methods while providing superior performance.

Our LS and LCP methods outperform the existing methods on 7 and 8 of the 10 languages with available MultiLS datasets, respectively. \textsc{G-Scale} has the potential to be applied to other tasks of rating a text input on a continuous scale using LLMs. We evaluated ablations of both of our methods.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Lexical Simplification and Automatic Text Simplification}
\label{sec:ls-ats}

LS is one of the many approaches to ATS. Other approaches operate at the sentence, paragraph, or text level, allowing deletions or rewriting of larger text spans, with the common goal of making the text easier to understand. Datasets that enable the evaluation, and to some extent, the fine-tuning of sentence simplification models for Japanese have recently become available (see \autoref{sec:ats-datasets}). Furthermore, recent LLMs can, in principle, be employed to simplify text even without fine-tuning \cite{brown_etal_2020_language}, and thus without extensive parallel data. This development raises a question: Should we abandon LS as an obsolete task and limit our focus to more general text simplification systems?

Because LS is both an indispensable operation for any ATS system and a practically useful task by itself, we believe that datasets and methods for LCP and LS are still an important research area complementary to higher-level ATS methods:


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Practical utility.}
Vocabulary knowledge has been recognized as one of the strongest factors, and possibly the strongest, in reading comprehension \cite[Ch.~11]{grabe_yamashita_2022_reading}. LS can be used as a tool to assist language learners while still displaying the original text. Even without the LCP component, LS can be used for on-demand simplification or glossing of words using a simple user interface. In conjunction with LCP, words to be simplified can be selected automatically based on language proficiency or the particular learner's needs (personalized LCP). Both of these modes of use, showing simplifications as glosses and adjusting the simplification level using lexical complexity thresholds, allow balancing comprehension with opportunities to learn.\footnote{Commercially available Amazon Kindle e-readers have a feature called Word Wise, offering ``simple definitions and synonyms displayed inline above more difficult words while you read'' (\url{https://www.amazon.com/gp/help/customer/display.html?nodeId=201645250}) based on the desired simplification level, which demonstrates the desirability of this function. Word Wise, however, does not employ LS and requires publishers to supply the glosses and levels.}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Detailed data.}
Both LS and LCP datasets provide detailed word-level data, consisting of multiple simplifications and complexity ratings, which are not available in sentence simplification or readability datasets. Such data has been used for analyses \cite{gooding_etal_2021_word,nohejl-etal-2024-difficult,saggion-etal-2024-lexical}, leading to the recognition of the needs of various target populations or problems in human-created simplifications. The insights from analyses of LS and LCP data are useful for any kind of ATS systems, as they inevitably perform simplifications at the lexical level as well.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Meaning preservation.}
Meaning preservation is a general challenge for all ATS systems, but it particularly affects systems operating at the sentence level or higher, which employ deletion or summarization \cite{agrawal-carpuat-2024-text}. LS is well-positioned to preserve the meaning of the original text. See \autoref{sec:error-analysis} for an analysis of our method in terms of errors in meaning preservation and other areas.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Work}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Japanese LS and LCP Datasets}
\label{sec:datasets}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Lexical simplification.}
Two LS datasets are available for Japanese: The \SNOW dataset
\cite{kajiwara-yamamoto-2015-evaluation} is based on newspaper articles and restricts target words to lemma form, although verbs and adjectives are inflected in Japanese. 
The Controlled and Balanced Dataset for Japanese Lexical Simplification (\CBD) \cite{kodaira-etal-2016-controlled} is based on the Balanced Corpus of Contemporary Written Japanese (BCCWJ) \cite{maekawa_etal_2014_balanced}. \CBD follows a more realistic setting, allowing not only target words in any word form but also substitutions spanning the adjacent \emph{fuzokugo} (dependent words).\footnote{See Appendix \ref{sec:glossary} for an English--Japanese glossary of linguistic terms. The authors of \CBD \cite{kodaira-etal-2016-controlled} mention only  ``particles''; however, in the actual data, other parts of speech are included in the substitutable context as well, all of which may be categorized as \emph{fuzokugo}, e.g., the whole sequence だったら\emph{撤退}しなければならないので including copulas, auxiliaries, and light verbs can be substituted.} Both datasets consist of approximately 1,500 simplifiable instances (target words in sentences), covering around 200 unique target words, and provide a list of simplifications for each instance but no complexity ratings.

Neither of the two LS datasets specifies what population (e.g., non-natives or children) it targets, although it has been shown that non-native speakers have different needs from natives, further differentiated by their proficiency level and native language \cite{paetzold-specia-2016-understanding}. Both datasets employed five crowdsourced annotators per instance, with \SNOW reporting no details and \CBD specifying only that annotators have completed 95\% of their previous assignments correctly and were native speakers.
The \SNOW dataset is available in full text, although redistribution is prohibited, and the \CBD dataset is distributed in a stand-off format that requires paid access to BCCWJ to reconstruct the complete data.

\Rev*{B2a}{In contrast to these two available datasets, our dataset, MultiLS-Japanese, targets non-native Japanese speakers and is available under a permissive license, namely, the Creative Commons Attribution-ShareAlike (CC-BY-SA).}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Lexical complexity.}
The only lexical complexity dataset for Japanese, Dataset of Japanese Lexical Complexity for Non-Native Readers (\JaLeCoN) \cite{ide-etal-2023-japanese}, provides complexity ratings by non-native speakers for single words and multiword expressions (MWEs). It consists of separate complexity ratings from annotators with Chinese or Korean L1 background and annotators with other L1 backgrounds, addressing the considerable advantage of the former in Japanese reading comprehension. The study \cite{ide-etal-2023-japanese} showed that words of Chinese origin or containing Chinese characters, which form a large part of the Japanese vocabulary, result in increased perceived complexity for annotators other than those with a Chinese or Korean L1 background. The dataset is based on texts from news and governmental press conferences and is available in full text for non-commercial purposes. It comprises around 18,000 words of running text, all of which were annotated with complexity ratings. As a natural result of this annotation scheme, the vast majority of the words have zero complexity, and the rest are heavily skewed to the low complexity levels \cite{ide-etal-2023-japanese}.

\Rev*{B2b}{Our dataset, MultiLS-Japanese, also provides lexical complexity annotation for non-native speakers but aims to provide higher-quality ratings for a smaller number of words by explicitly annotating selected target words, as opposed to asking annotators to annotate complex words in a running text,  skipping simple words at their discretion. Moreover, in contrast to JaLeCoN, which provides data only for LCP, MultiLS-Japanese has gold labels for both LCP and LS based on the MultiLS framework \cite{north-etal-2024-multils},
enabling consistent evaluation of the whole LS pipeline.}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Other Japanese ATS Datasets}
\label{sec:ats-datasets}

LS is a specific task within the broader area of ATS. Another common ATS task is sentence simplification, and more recently, paragraph- and document-level simplification tasks have been studied as well.

In addition to LS datasets, there are several Japanese datasets for sentence simplification: An early news-based dataset \cite{goto-etal-2015-japanese} is not publicly available. The SNOW T15 \cite{maruyama-yamamoto-2018-simplified} and SNOW T23 \cite{katsuta-yamamoto-2018-crowdsourced} datasets provide simplifications of sentences from the Tanaka corpus, a collection of short example sentences from language textbooks.
JADES \cite{hayakawa-etal-2022-jades} consists of simplifications of 3,907 sentences from news articles targeting non-native speakers. MATCHA \cite{miyata_etal_2024_matcha}, which is based on articles from a website for international visitors to Japan,\footnote{\url{https://matcha-jp.com}} provides 16,000 sentence simplifications targeting non-native speakers. JASMINE \cite{horiguchi-etal-2024-evaluation} is a sentence-level medical domain evaluation dataset. JASMINE and the SNOW datasets focus on word- or phrase-level substitutions, forming an intermediate step between LS and sentence simplification, whereas JADES and MATCHA employ larger-scale rewriting operations.

All these datasets share the common features of being sentence-level and providing a single gold simplification. JADES and MATCHA target non-native speakers at the proficiency level corresponding to the Japanese Language Proficiency Test (JLPT) level N4, equivalent to level A2 \cite{jees_2025_indication} in the Common European Framework 
\pagebreak
of Reference for Languages (CEFR). While MATCHA is larger, JADES has been manually annotated for operations (e.g., \textsc{replace}, \textsc{delete}, and \textsc{split}), enabling easier analysis.


Compared with JADES and MATCHA \cite[also see \autoref{sec:ats-datasets}]{hayakawa-etal-2022-jades,miyata_etal_2024_matcha}, our dataset targets more proficient speakers. JADES and MATCHA target relative beginners (JLPT N4), who need more aggressive simplification, involving deletions, splitting, or rewriting. At JLPT levels N1 and N2, which we target, we hypothesize that LS is sufficient to achieve a very high level of comprehension. Although our dataset is smaller than JADES or MATCHA, it covers a variety of genres and topics, whereas JADES and MATCHA are each based on a single source. The sentence simplification datasets and our dataset complement each other by providing data for different tasks (as discussed in \autoref{sec:ls-ats}) and targeting different proficiency levels.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Lexical Simplification}

The early approaches to LS were rule-based \cite{devlin_tait_1998_use,debelder_moens_2010_text}. Data-driven approaches were soon applied \cite{yatskar-etal-2010-sake,biran-etal-2011-putting,horn-etal-2014-learning} to English, where a large simplified corpus, Simple English Wikipedia,\footnote{\url{https://simple.wikipedia.org}} is available. Later methods that did not require rules, databases, or simplified corpora were based on static word embeddings \cite{glavas-stajner-2015-simplifying} and, more recently, on masked language models (MLMs) \cite{qiang_etal_2020_lexical,qiang_etal_2020_lsbert}.

The current state-of-the-art methods for LS leverage LLMs such as OpenAI's GPT-3 \cite{brown_etal_2020_language} and \Rev*{A3a}{GPT-4 \cite{openai_etal_2024_gpt}}.
An ensembled prompting method using GPT-3 \cite{aumiller-gertz-2022-unihd} achieved a top result for English, Spanish, and Portuguese in a shared task. A derived method \cite{enomoto-etal-2024-tmu} using GPT-4 achieved the best overall result for ten languages in the Multilingual Lexical Simplification Pipeline (MLSP) shared task \cite{shardlow-etal-2024-bea}, where Japanese was evaluated on \MLSJ\Rev*{A2}{ (our dataset introduced in \linebreak
\autoref{sec:dataset}}). This method is based on aggregating results from multiple LLM inferences, thus using significant resources to simplify each word.

\Rev*{B2c}{Our LS method is also based on LLM prompting but uses a single LLM inference while improving accuracy.}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Lexical Complexity Prediction}
\label{sec:related-lcp}

Word frequency has long been used as the variable most predictive of lexical complexity, particularly the frequency in film or YouTube subtitles \cite{shardlow-2013-comparison,nohejl-etal-2025-film}, which approximates the spoken language. Lexical dispersion measured by the number of YouTube channels or videos containing a word was demonstrated to be an even stronger predictor \cite{nohejl_watanabe_2025_dispersion}. Various other word-level features have been used as well; however, their applicability (e.g., for stress patterns) and availability (e.g., for psycholinguistic data) vary based on language \cite[Sec.~3 lists 20 features possibly applicable to English]{shardlow_etal_2022_predicting}. \linebreak
\Rev*{A3b}{Shardlow et al.}\ also emphasized that context is an important factor of lexical complexity because it determines in which sense a polysemous word is used and can add to complexity if a word is used in an unexpected context.

In feature-based models, attempts were made to model the interaction with context using $n$-gram probability of the immediate context of the target word or MLM-based surprisal (to model the unexpectedness of the word) and features based on dependency parsing. The results were mixed. Contextual surprisal was found to have a minimal effect \cite{tack_2021_mark}. Two shared tasks \cite{paetzold-specia-2016-semeval,yimam-etal-2018-report} were dominated by feature-based systems focused on word-level features. Features that depend on context were limited to either only $n$-grams \cite{paetzold-specia-2016-sv000gg}, without further analysis of the feature's contribution, or only dependency-based features \cite{gooding-kochmar-2018-camb}, but in that case, dependencies were not among the top ten contributing features.

Later, using fine-tuned MLMs 
became the dominant approach in cases where enough training data was available \cite{shardlow-etal-2021-semeval}. 

In the MLSP shared task \cite{shardlow-etal-2024-bea}, where lexical complexity was predicted in ten languages, the top-performing systems across all languages were, depending on the metric used, TMU-HIT \cite{enomoto-etal-2024-tmu} and Archaeology \cite{cristea-nisioi-2024-archaeology}. TMU-HIT was based entirely on prompting LLMs, 
whereas Archaeology was based entirely on word-level features. A study \cite{smadu-etal-2024-investigating} comparing LCP using feature-based models, fine-tuned MLMs, and LLMs concluded that the computationally intensive methods based on LLMs barely outperform the earlier, more lightweight methods.

\Rev*{B2d}{The method we propose for LCP is also LLM-based, but it reduces the computational cost compared to TMU-HIT while providing more accurate predictions. Our method can also combine LLM predictions with features, such as frequency, leading to further improvements in accuracy.}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Dataset: \MLSJ}
\label{sec:dataset}

Japanese is well known to be difficult to learn as a second language \cite{grainger_2005_second,foreignserviceinstitute_2025_foreign}. Attaining reading or writing proficiency is particularly demanding because it involves mastering its complex writing system. Yet, no Japanese LS dataset targeting non-native speakers is available. To fill this gap, we constructed a unified LS and LCP dataset targeting non-native Japanese speakers.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[b]
\input{04table01.tex}
\hangcaption{Comparison of Japanese LS and LCP datasets.For all datasets, we count only simplifiable words as ``LS Instances.'' ``CK'' stands for Chinese or Korean L1.\quad\tblfn{1} Self-reported.\quad\tblfn{2} Holders of JLPT (N)1, N(2).\quad\tblfn{3} Details in Japanese: \protect\url{https://www.jnlp.org/GengoHouse/snow/e4}\quad\tblfn{4}~Annotation in stand-off format, requires paid copyrighted BCCWJ data.
}
\label{tab:cmp}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Overview}
\label{sec:dataset-overview}

\MLSJ is the first Japanese LS dataset targeting non-native speakers. More specifically, it targets non-native Japanese speakers whose first language is neither Chinese nor Korean. The data consists of 600 instances, where each instance is a target word in a sentence context, for which a lexical complexity value and simpler substitutions are provided.
\Rev[rev:A4]{A4, also see Table 1}{\autoref{tab:cmp}} compares \MLSJ with the available Japanese datasets for LS and LCP. While MultiLS-Japanese consists of a relatively small number of instances, it employs more annotators per instance and provides lexical simplifications for more unique words (types) than the previous datasets. For LCP, 572 of the 600 annotated instances in MultiLS-Japanese have non-zero mean complexity rating, in contrast with JaLeCoN, which is larger, but only a small fraction of the annotated words have non-zero complexity. This is a result of a different annotation scheme: JaLeCoN provides ratings for all words in a running text, whereas MultiLS-Japanese provides ratings only for selected target words. 

For lexical complexity, a mean value and individual ratings by ten annotators are provided. The annotators were holders of JLPT levels N1 or N2, or their older equivalents 1 and 2. These two highest levels of JLPT are often the minimum requirement for employment or university acceptance \cite{jasso_2024_universities} and correspond approximately to CEFR levels B2 and C1 \cite{jees_2025_indication}. JLPT has been criticized for not testing productive skills \cite{nishizawa_etal_2022_review}; however, that actually aligns with our task.

For LS, we provide an aggregated list of substitutions as well as individual substitutions from ten annotators. The annotators for LS were native Japanese speakers, with at least one year of experience teaching Japanese as a foreign language, instructed to provide simplifications for language learners without assuming a particular native language.

For lexical complexity, we target non-native speakers whose native language is neither Chinese nor Korean. As the two languages largely overlap in lexicon with Japanese, Japanese LS targeting Chinese or Korean (CK) L1 speakers should be evaluated separately. This was reflected in differences between CK and non-CK complexity ratings in \JaLeCoN \cite{ide-etal-2023-japanese}. The systematic difference in Japanese lexical complexity perception by Chinese L1 speakers was further confirmed by a reannotation and analysis of a subset of the present dataset, while no such difference was observed for a group of Czech and Slovak L1 speakers \cite{nohejl-etal-2024-difficult}.


We split the dataset into a trial set (30 instances) and test set (570 instances), which share a similar word frequency, lexical complexity, and part of speech (POS) distribution \cite{nohejl-etal-2024-difficult} to make the trial set representative and suitable for training simple models or for in-context learning. As we explain in \autoref{sec:selection}, the targets co-occur in the same sentence by groups of three. Therefore, the trial set comprises 10 sentences, while the test set comprises another 190 sentences. 

The gold LS and LCP labels for the trial set were published on the open web for the MLSP shared task \cite{shardlow-etal-2024-bea}.
We therefore recommend using only the test set to evaluate models whose training data could have been contaminated. We made the full MultiLS-Japanese dataset\footnote{
\url{https://huggingface.co/datasets/naist-nlp/multils-japanese}} and accompanying files (annotator profiles and annotation instructions)\footnote{\url{https://github.com/naist-nlp/multils-japanese}} available online. We request users not to disseminate the gold labels on the open web. The format of the dataset is described in Appendix \ref{sec:format}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{MultiLS Framework}
\label{sec:dataset-multils}

The MultiLS framework \cite{north-etal-2024-multils} specifies that lexical simplifications and lexical complexity ratings are given for the same target words in the same contexts; hence, it is \textbf{multitask}, whereas previous datasets handled only one of these tasks. Furthermore, MultiLS datasets for ten diverse languages are currently provided, all available in a common format via Hugging Face Hub as MLSP2024,\footnote{\url{https://huggingface.co/datasets/MLSP2024/MLSP2024}} making MultiLS \textbf{multilingual}.
The ten datasets are of comparable size and use the same annotation protocol \cite{shardlow-etal-2024-extensible}. Articles detailing the Portuguese data \cite{north-etal-2024-multils}, Spanish and Catalan data \cite{saggion-etal-2024-lexical}, and Italian data \cite{occhipinti_2024_introducing} have been published. In the following sections, we focus on the Japanese dataset, including its previously unpublished parts. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Further Goals and Additional Data}

In addition to providing annotation for both LCP and LS, \MLSJ fulfills the following goals that are orthogonal to the MultiLS framework: 
\begin{enumerate}
    \item varied sources representative of texts read by non-native speakers (\autoref{sec:sources}), 
    \item appropriate handling of Japanese morphology and MWEs (\autoref{sec:morpho}),
    \item sampling of target words representative of multiple factors of complexity (\autoref{sec:selection}),
    \item minimization of external influences on perception of lexical complexity   (\autoref{sec:sel-sentences}),
    \item a single set of annotators who annotate each instance, enabling individual data analysis, 
    \item a permissive license for the full dataset to facilitate further analysis or extension.
\end{enumerate}

In addition to the data in the common MultiLS format, we also release individual annotator profiles, individual annotations, complete annotation instructions (see \autoref{sec:annotation}), detailed automatic and manual tags, and metadata for each dataset instance (see Appendix \ref{sec:format}).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Source Texts}
\label{sec:sources}

A common approach to constructing LS and LCP datasets has been to use texts from a single source---e.g., Wikipedia \cite[LexMTurk dataset]{horn-etal-2014-learning} or  newspaper \cite[\SNOW dataset]{kajiwara-yamamoto-2015-evaluation}---or only a few genres, e.g., Wikipedia and news \cite[CWIG3G2 dataset]{yimam-etal-2017-cwig3g2}. This may poorly represent texts read by the target population and severely bias the language towards certain registers or topics.

While Wikipedia is at least relatively diverse in terms of topics and offers favorable licensing terms, relying solely on Wikipedia would be problematic for Japanese, where a large part of the content is created through translation from English \cite{hautasaari_ishida_2011_discussion}, potentially leading to underrepresentation of local topics or culturally dependent vocabulary. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 2
\begin{table}[b]
\vspace{-0.5\Cvs}
\input{04table02.tex}
\hangcaption{Sources of \MLSJ by category. The dataset is composed of a total of 200 sentences. Licenses: \faCreativeCommons{} are Creative Commons licenses compatible with CC-BY-SA, \faCreativeCommonsPd{} is public domain. \quad\tblfn{1} Only pages about locations in Japan to avoid translated content. \quad\tblfn{2} Selected works of Dazai Osamu (\dag 1948), Umezaki Haruo (\dag 1965), Tanizaki Jun'ichir\={o} (\dag 1961), Yoshikawa Eiji (\dag 1955), Sakaguchi Ango (\dag 1955), Edogawa Ranpo (\dag 1965), Yamamoto Sh\={u}gor\={o} (\dag 1967), and Yanagita Kunio (\dag 1962) published 1945--1965.}
\label{tab:sources}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Instead, we strove to sample sentences from diverse genres and topics, emphasizing sources that Japanese non-native speakers may read. The sources constituting the final dataset are shown in \autoref{tab:sources}. We carefully selected the sources such that the complete dataset could be distributed under the CC-BY-SA 4.0 International License,\footnote{\url{https://creativecommons.org/licenses/by-sa/4.0/}} which was the main limiting factor for source selection. All sources had to be released under a compatible Creative Commons license or to be in the public domain. Although we prioritized recent texts, we felt that the inclusion of literary fiction is important for representativeness and justifies a limited amount of slightly dated language. We have therefore included several works with expired copyrights published by well-known authors in the years 1945--1965.\footnote{
    Until 2019, the Japanese copyright law protected works for only 50 years after the author's death. 
}

We considered but decided not to use Tatoeba,\footnote{\url{https://tatoeba.org}} which provides examples for language learners as opposed to naturally occurring sentences, and Japanese Wikiversity,\footnote{\url{https://ja.wikiversity.org}} which was available only as a beta version when we were collecting data. We also could not use Japanese WikiHow\footnote{\url{https://www.wikihow.jp}} because its CC-BY-NC-SA license is incompatible with the CC-BY-SA license.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Target Words and Japanese Morphology}
\label{sec:morpho}

LS occurs at the level of words. However, this does not mean that we are merely substituting one lexeme for another: we are substituting word forms so that the sentence remains grammatical after the substitution. Some datasets make this task artificially simpler. For instance, the Portuguese LS dataset SIMPLEX-PB \cite{hartmann_etal_2018_simplex} contains substitutions only in lemma form (e.g., \emph{coletaram}, past, 3rd pers.\ pl.\ $\rightarrow$ \emph{recolher}, inf.), and the Japanese \SNOW \cite{kajiwara-yamamoto-2015-evaluation} contains only target words in uninflected form. The former approach results in ungrammatical simplified sentences, while the latter restricts the dataset to a small subset of language; therefore, we avoided both.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[b]
\input{04table03.tex}
\hangcaption{Three common units of segmentation, SUW, LUW, and \emph{bunsetsu}, when simplifying the verb in the sentence 彼女は読書している (She is reading a book). Dashed line marks an optional (traditional) \emph{bunsetsu} division. Abbreviations in glosses: TOPic, ACCusative, GERund, PReSent.}
\label{tab:seg}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

In the case of the Japanese language, we need to deal not only with inflections but also with segmentation. Multiple units can be considered a word for the purposes of LS. Japanese script does not separate words in writing, and in Japanese NLP and linguistics, sentences are commonly segmented into short unit words (SUWs), long unit words (LUWs), or \emph{bunsetsu} \cite{omura-etal-2021-word}. \autoref{tab:seg} illustrates that targets consisting of a single SUW or LUW would not be sufficient for LS, as verbs generally cannot be simplified by substituting a single SUW or LUW. While a \emph{bunsetsu} is long enough to support verb substitution, it would also include many dependent words that are unnecessary for simplification, e.g., particles with nouns. Therefore, we decided to target such sequences of SUWs that are long enough to substitute the given word. The resulting target sequences span at most one \emph{bunsetsu}, except in cases where a longer MWE needs to be simplified as a single unit (e.g., 人目を引く ``to draw attention''). For SUW segmentation, we used MeCab \cite{kudo-etal-2004-applying}, a Japanese morphological analyzer, and the UniDic 2.1.2 dictionary \cite{den_etal_2007_development}, which is distributed as \texttt{unidic-lite}. 
The precise guidelines for delimiting the boundaries of target words and an overview of MWEs in the dataset are given in Appendix \ref{sec:guidelines}.

Our approach slightly differs from both \JaLeCoN and \SNOW. The approach used for \JaLeCoN \cite{ide-etal-2023-japanese} is mostly based on SUWs and manually selected MWEs, thus typically preferring smaller units. This does not pose practical problems for LCP, but as we have shown in \autoref{tab:seg}, it is insufficient for substituting verb forms. \CBD \cite{kodaira-etal-2016-controlled}, on the other hand, uses larger spans including all dependent words surrounding the target word on both sides (as discussed in \autoref{sec:datasets}), which generally extend over two \emph{bunsetsu}. When inspecting the dataset, we found that dependent words on the left side of the target words were rarely altered by the substitutions. This was the expected result because dependent words in the Japanese language follow the corresponding independent words rather than precede them.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Selection of Target Words and Sentences}
\label{sec:selection}

To better control and partially automate the sampling of suitable sentence contexts and target words, we adopted a scheme in which sentences with a single target word were first sampled automatically from the source texts. From this sample, we have manually selected 200 sentences to constitute the final dataset, manually picking two additional target words in each sentence for a total of 600 target words. We refer to the automatically tagged words as primary and to the manually selected ones as secondary.

Simplifying or predicting complexity for three words per sentence has the following advantages compared to the common format of LS datasets with a single target word per sentence:
\begin{enumerate}
    \item It reduces the amount of text to be read, enabling annotators to concentrate better.
    \item It reflects that simplifying a single word in a sentence is often insufficient. 
    \item It allows us to analyze how lexical complexity is affected by other words in the sentence.
\end{enumerate}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Automated pre-selection of words}
\label{sec:sel-words}

With a relatively small dataset and many factors affecting lexical complexity, we faced the risk of not representing some of the factors if we sampled the words randomly.
Therefore, the primary target words were selected to represent diverse word frequencies, character compositions, and parts of speech. Additionally, the primary targets include specific categories of words known to be difficult for learners: compound verbs \cite{himeno_1999_fukugo,ninjal_2025_compound}, complex auxiliaries, complex particles (together covering most relatively advanced function words), and mimetics \cite{yoshioka_2016_acquisition}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1
\begin{figure}[b]
\begin{center}
\includegraphics{32-4ia4f01.eps}
\end{center}
\hangcaption{Selection of primary target words from a Cartesian product of word frequencies, character compositions, and parts of speech. \# is the number of words with the given properties to be selected.}
\label{fig:word-selection}
\vspace{-0.5\Cvs}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We partitioned candidate word properties in a discrete space with three axes: word frequency, character composition, and POS, as shown in \autoref{fig:word-selection}. For each element of the space (e.g., nouns written in \emph{katakana} with $\log_{10}$ frequency from $-4.5$ to $-4.0$), we pre-selected candidate words and sentences containing them, and determined a number to be selected manually from them to achieve an approximately even distribution of target words in the space, given the number of words with each property.

The three groups of parts of speech are composed as follows:
\begin{itemize}
\item \textsc{Nouns} (90): nouns except proper nouns and verbal nouns (VN)+\emph{suru} (i.e., \emph{sahen} verbs).
\item \textsc{Verbs} (50): VN+\emph{suru} and independent verbs except verb+verb compound verbs.
\item \textsc{Other} (60):
    verb+verb compound verbs (10);
    prefixes (5);
    suffixes (5);
    adjectives (10);
    adjectival nouns (10);
    non-mimetic adverbs (7);
    mimetic adverbs (3);
    particles, including complex (5); and
    auxiliaries, including complex ones (5).
\end{itemize}
The division into the three groups is only a technical means to ensure approximately even distribution of the two most important groups (\textsc{Nouns} and \textsc{Verbs}) across all combinations of word frequencies and character compositions. At the automatic pre-selection stage, the word candidates consist of a single SUW---except for VN+\emph{suru} (e.g., 活躍する), complex particles  (e.g., にもかかわらず), and complex auxiliaries (e.g., ねばならぬ).\footnote{The verb+verb compound verbs generally consist of a single SUW.} These compound word categories are an important part of Japanese vocabulary and include words difficult for learners. Without this special handling, it would be impossible to automatically select them based only on the SUW POS. On the other hand, the prefixes and suffixes are single SUWs, but we manually extended the selected targets to the whole words (e.g., 尽くめ \textrightarrow{} 黒尽くめ), except for cases of very productive, easy-to-substitute affixes (e.g., the temporal prefix 翌 or the honorific suffix 氏; see \appref{sec:g-cd}). We used a custom version of the Japanese TUBELEX corpus \cite{nohejl-etal-2025-film} to determine word frequencies, including those of the aforementioned multi-SUW words. 

We determined the order of the most frequent 2,500 \emph{kanji} based on Japanese Wikipedia. We opted for Wikipedia and a limit (2,500) slightly higher than the requirements of JLPT N1 or the number of the \emph{j\={o}y\={o} kanji},\footnote{As of 2025, there are 2,136 \emph{j\={o}y\={o} kanji} ``\emph{kanji} for regular use'', promulgated by the Japanese Ministry of Education, Culture, Sports, Science and Technology.} to enable the selection of relatively specialized or less frequent vocabulary if it satisfies the frequency criteria of our selection. 
%%%%\Rev*{comm-B1a}{See \hyperref[sec:limitations]{Limitations} for further discussion of \emph{kanji} and complexity}. 
    See Limitations for further discussion of \emph{kanji} and complexity. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Automated pre-selection of sentences}
\label{sec:sel-sentences}

Other than containing the primary target word, the sentences automatically pre-selected from the source texts had to fulfill the following criteria to minimize external influences on the perception of lexical complexity:
(1) The sentence is 16--32 SUWs and 24--60 characters long.
(2) The sentence does not contain any \emph{kanji} outside the top 2,500 most frequent \emph{kanji} in Japanese Wikipedia. (3) The primary target word is the lowest frequency word in the sentence.

We restricted the sentence length to reduce the effect that a long, complicated sentence could have on the judgments about lexical complexity, and we used a range in which we expected both to find two more simplifiable words and to cover most of the high-complexity words for our target population by the three target words. The relatively short sentences also enabled the annotators to concentrate better on their task. We also avoided rare \emph{kanji} or words rarer than the target words in order not to affect the lexical complexity ratings.

At the end of this stage, we had pools of candidate primary target words in sentences, each corresponding to a combination of word properties (frequency, character composition,\footnote{The main classes of characters of the Japanese script are: \emph{hiragana}, a basic syllabary; \emph{katakana}, a syllabary used mostly for foreign-origin words from Western languages and mimetic words, and \emph{kanji}, Chinese characters used mostly for content words. In particular, \emph{kanji} present a cognitive and memorization challenge for learners.
} and the POS group) and coupled with the required number of items to be manually selected.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Manual selection of sentences and words}

The authors of the paper manually selected the required number of sentences from each pool, choosing two secondary target words from each and adjusting the extent of all target words to the appropriate morphological boundary as outlined in \autoref{sec:morpho}. Each selection was initially conducted by a native Japanese speaker and reviewed by a non-native Japanese speaker, or vice versa, and corrected if necessary until we completed the selection of 200 sentences with 1 primary and 2 secondary targets each, fulfilling the following criteria:
\begin{itemize}
\item A missing context does not complicate the comprehension of individual words or the sentence.
\item Each target word is simplifiable and spans the appropriate SUW segments.
\item Each target word in the dataset is unique.
\item No single source contributes more than 50\% to the entire dataset.
\end{itemize}


As the least frequent word in the sentence, we expect the primary target to be among the most difficult words in the sentence. Among the remaining words, we selected as secondary targets the two that we expected to be most difficult for our target population. While our judgment may not perfectly match our target population's aggregate perception, due to the relatively short length of the sentences, we expect the mismatches to be relatively rare and to affect sentences where multiple candidates of similar complexity are available for the third target word. \autoref{tab:ex-selected} shows examples of the selected sentences and target words along with two instances of words of similar complexity that were not selected, i.e., cases where we may have not selected the most complex words as targets. The examples also illustrate target words of various parts of speech and character compositions.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 4
\begin{table}[t]
\input{04table04.tex}
\hangcaption{Examples of sentences and target words (bold) selected for the dataset. Other words of complexity comparable to the selected target words are underlined. Rather than being literal, the translations illustrate the complexity of the original words in English. ``\emph{Kata.}+\emph{hira.}''\ stands for \emph{katakana}+\emph{hiragana}.}\label{tab:ex-selected}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Annotation and Review}
\label{sec:annotation}

\Rev*{B3a}{Annotating lexical complexity is a relatively straightforward task, which relies on the subjective judgments of the annotators from the target population. On the other hand, the annotation of appropriate lexical substitutions poses a conundrum: annotators from the target group cannot annotate appropriate substitutions when the text is difficult for them to understand in the first place. We addressed this problem by working with two separate annotator groups.}

We hired two groups of annotators through an intermediary company, which allowed us to recruit annotators fulfilling our requirements and to ensure quality control. The ten annotators performing lexical complexity annotation were non-native Japanese speakers who had JLPT levels N2 or N1 (or the older equivalents 2 and 1) and whose first language was neither Chinese nor Korean. The ten annotators performing LS annotation were native Japanese speakers with at least one year of professional experience in teaching Japanese as a foreign language\Rev*{B3b}{ and were instructed to provide simplifications for language learners without assuming a particular native language.
Therefore, in the case of LS, we were relying on the professional experience of the annotators and their ability to follow these instructions}.

Each annotator received instructions for their task and a profile questionnaire. Non-native speakers were given instructions and questionnaires in Japanese appropriate for their proficiency level and in English. Appendix \ref{sec:inst} provides the full instructions and questionnaires along with English translations for cases in which the original texts were only in Japanese. The corresponding files are in the project's online repository.

We asked both groups to annotate the first ten sentences, which constitute the trial data of the final dataset. We evaluated the initial annotation both manually and by computing agreement measures between individuals and the rest of the group. We did not find any outliers or errors that would prevent us from using the annotation or would require us to replace any annotators.

In the case of lexical simplifications, we gave each annotator individual feedback on their annotation, generally encouraging them to maintain the good quality of their work and pointing out areas for improvement in the annotation. We also supplied more examples to all annotators, illustrating how we expected the instructions to apply to the items they had already annotated (provided in \autoref{fig:ls2} in \appref{sec:inst-ls}). 
In the case of one LS annotator, in whose annotation issues with simplicity or meaning preservation occurred repeatedly, we reviewed their work once more after they completed another ten sentences, to ensure that they understood the feedback.

We reviewed the annotation of the entire dataset automatically for formal errors (e.g., duplicate simplifications or spurious characters) and agreement with other annotators (we did not detect any outliers or suspiciously similar annotations). We also performed character normalization and manual corrections based on automated screening to correct unambiguous errors such as ungrammatical word forms, as detailed in Appendix \ref{sec:correction}. Finally, we reviewed the dataset for offensive language 
%%%%(see \hyperref[sec:ethical]{Ethical Considerations} for details).
    (see Ethical Considerations for details).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 5
\begin{table}[t]
\input{04table05.tex}
\hangcaption{Summary of the annotator profiles. The last seven rows report means and standard deviations separated by $\pm$. Full questions in Appendices \ref{sec:lcaq} and \ref{sec:lsaq}.}
\label{tab:profiles}
\vspace{-0.5\Cvs}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Each annotator was given a unique identifier so that their individual annotations could be matched to their profiles. We did not collect any personally identifying information. In addition to aggregated data, we released all of the individual annotations for both tasks and the annotator profile data. \autoref{tab:profiles} summarizes the annotator profiles.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Proposed Methods for LS and LCP}
\label{sec:methods}

The MultiLS-Japanese dataset, 
\pagebreak
presented in the previous section, was used for an evaluation of a shared task \cite{shardlow-etal-2024-bea}, in which systems representative of the state-of-the-art approaches participated. We therefore proposed new methods with the goal of improving both accuracy and efficiency over the shared task results, not only on MultiLS-Japanese but across all languages with available MultiLS datasets.

The current state-of-the-art methods for LS and LCP require multiple LLM inferences to simplify a single word or predict its complexity. In contrast, we propose methods for both subtasks that use only a single LLM inference per data instance. Our methods use the same prompt across all languages, and for LCP we employ a novel calibrated token-probability-based scoring technique, \textsc{G-Scale}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Lexical Simplification}
\label{sec:method-ls}

In LS, multiple substitutions are generated for target words in a sentence context. The ordered set of substitutions is then evaluated against an ordered set of gold substitutions. In the general case, the goal is to match both the gold substitutions and their order.

The system UniHD \cite{aumiller-gertz-2022-unihd}, which won the TSAR 2022 shared task, ensembled six combinations of prompts and temperatures, combining their outputs using rank average. In all cases, the temperature was non-zero. As motivations for this approach, the authors cite safeguarding against malformed outputs (the chance of multiple outputs being discarded due to automatic filtering is lower) and a higher diversity of substitutions.
The system TMU-HIT \cite{enomoto-etal-2024-tmu} used an approach derived from UniHD but ensembled ten instances of the same prompt and temperature.

We simplified the approach by using a single prompt and greedy decoding (by setting temperature to zero), as we hypothesized that the safeguarding against malformed outputs \cite{aumiller-gertz-2022-unihd} was no longer necessary on more recent LLMs and we questioned whether sampling multiple responses to the same prompt \cite{enomoto-etal-2024-tmu} provides any advantage over greedy decoding.
Our prompt (see \appref{sec:ls-prompt}) is based on the three conditions that lexical simplifications must fulfill (i.e., grammaticality, meaning preservation, and simplicity). Additionally, we specifically mention simplicity for ``language learners,'' which are targeted by many of the MultiLS datasets.

\Rev*{A6a}{Note that our method does not involve any reranking: it simply outputs simplifications in the order in which they were generated by the LLM. The effectiveness of this approach was previously demonstrated in the TMU-HIT system \cite{enomoto-etal-2024-tmu}, where it outperformed two reranking approaches. While we did not experiment with reranking, we compared ranking based on a single LLM output (our method) with ordering by mean rank from multiple outputs (TMU-HIT) and ordering by median rank, as discussed in \autoref{sec:experiments-ls}.}

We conducted a few preliminary experiments with different prompt wording on the English and Japanese trial sets: We observed that using complex words such as ``synonym'' or ``substitute'' in the prompt led to more complex words appearing near the top of the generated simplification lists, and therefore, we attempted to limit them. Additionally, we saw an improvement in the quality and consistency of formatting on Japanese after including the identifiers S, X, and Y for the sentence, target word, and simplifications, respectively.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Lexical Complexity Prediction}

In LCP, a complexity score is predicted for target words in a sentence context. The gold complexity score is an average of ratings by annotators from the target population. Therefore, although the annotators rate complexity on a discrete scale, the prediction methods must predict values on a continuous scale.

We propose a new method, \textsc{G-Scale}, applicable to any tasks of rating a text input on a continuous scale. The method is motivated by the following observations about existing LCP methods based on LLM prompting \cite{enomoto-etal-2024-tmu,smadu-etal-2024-investigating}:
\begin{enumerate}
    \item \textbf{Low efficiency:} The current state-of-the-art LLM-based LCP methods are extremely computationally intensive, using 20 inferences \cite{enomoto-etal-2024-tmu,smadu-etal-2024-investigating} per instance to obtain results competitive with simpler models.
    \item \textbf{Lack of calibration:} As the LLM-based LCP methods are based on weighting the scores by token probabilities, they depend on two conditions: (1) that the probabilities provide valuable information and (2) that they have the correct magnitude. Current methods do not perform any calibration, taking the probabilities at face value. 
    \item \textbf{Bad fit:} The LLM-based TMU-HIT \cite{enomoto-etal-2024-tmu} achieved a better correlation but a worse fit, i.e., a larger error, than a feature-based model.
\end{enumerate}

\Rev*{comm-B2b}{Although we were motivated by the weaknesses of existing LCP methods, we formulated our method so that it is potentially applicable to other tasks where a text input is rated on a scale, and the model is expected to accurately predict human ratings, not only to correlate with them, e.g., automated essay scoring.} We devised a scoring function and a training algorithm for \textsc{G-scale} by addressing the problems mentioned above:


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Efficient use of token probabilities.}
The existing LLM-based LCP methods \cite{enomoto-etal-2024-tmu,smadu-etal-2024-investigating} were derived from \textsc{G-Eval} \cite{liu-etal-2023-g}, which was proposed as a method for evaluating natural language generation (NLG) on a numerical scale via LLM prompting. \textsc{G-Eval}'s authors proposed weighting discrete scores by their token probabilities or, when the probabilities are not available, estimating the probabilities by sampling multiple responses. Only the latter option was used in LCP \cite{enomoto-etal-2024-tmu,smadu-etal-2024-investigating}. As token probabilities are now available even for the latest OpenAI models as \texttt{logprobs} in the API,\footnote{\url{https://platform.openai.com/docs/api-reference/completions/create}} we used probabilities, reducing the computation cost by a factor of 20. Furthermore, we predict only a single token, avoiding the generation of superfluous explanations.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Calibrated temperature scaling.}
\textsc{G-Eval} and the two LCP methods use the same weighting scheme, where token probabilities (or probability estimates) are used as weights for the scores:
\begin{equation}
    \textrm{score}_\text{\textsc{G-Eval}} = \sum_{i=1}^n p(s_i) \cdot s_i\label{eq:score-geval}
\end{equation}
where $s_i \in S$ are discrete scores from a scale $S$ (e.g., 1, 2, 3, 4, and 5), which the model is instructed to use in the prompt. Consequently, the distribution of $\textrm{score}_\text{\textsc{G-Eval}}$ depends on the token probability distribution. The distribution may be too spiky, i.e., assigning almost all probability to the top token, or too noisy, i.e., assigning the remainder of the probability randomly. The two LCP methods represent scores by one to two tokens (one token: 0, 1; two tokens: 0.25, 0.5, 0.75), which may further complicate the interpretation of the probabilities, and use the temperature values 0.7 \cite{enomoto-etal-2024-tmu} and 0.8 \cite{smadu-etal-2024-investigating} to sample tokens and estimate probabilities from them without explaining how the temperature values were determined.

We use single-token scores as in the original \textsc{G-Eval} and replace $p(s_i)$ in its formula with temperature-scaled softmax:
\begin{equation}
\textrm{score}_{\textrm{softmax}}^{(T)} = \sum_{i=1}^n \frac{\exp\left(\frac{\log p(s_i)}{T}\right)}{\sum_{j=1}^n \exp\left(\frac{\log p(s_j)}{T}\right)} \cdot s_i \label{eq:score-softmax}
\end{equation}

Note that in contrast with the previous methods, we do not use the temperature $T$ for decoding an output consisting of multiple tokens but for computing a score from a single token distribution using the formula above. We calibrate the value of $T$ by a search in a fixed set of values using a training set: First, we perform LLM inference using a prompt once for each instance of the training set, recording the token probability distributions. Then, we select a value of $T$ that maximizes the linear correlation between the scores predicted by \autoref{eq:score-softmax} and the gold scores of the training set.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Fitting a linear regression.}
To address the problem of a relatively large error despite a good correlation, we fit a linear regression model with $x_1 = \textrm{score}_{\textrm{softmax}}^{(T)}$ and optionally other features $(x_i)_2^k$ as independent variables. Because this can result in predicted values being out of the scale, we clip the result to the interval $[s_1, s_n]$, assuming that $s_1$ and $s_n$ are the lowest and highest discrete scores, respectively, of the scale $S$. This yields the final formula of the \textsc{G-Scale}'s scoring function:
\begin{equation}
\textrm{score}_\text{\textsc{G-Scale}}^{(T, \boldsymbol{\upbeta})} =
    \max\left(s_1,\, \min\left(s_n,\,
        \beta_0 +
        \beta_1 \cdot \textrm{score}_{\textrm{softmax}}^{(T)} +
        \sum_{i=2}^k \beta_{i} \cdot x_i
    \right)\right)
    \label{eq:g-scale}
\end{equation}
Linear regression also makes it possible to ensemble predictions from multiple prompts or models or to add features, e.g., word frequency in the case of LCP, for a hybrid approach combining feature-based modeling and LLM prompting. Depending on the additional features and the training set size, a more powerful model (e.g., support vector machine or random forests) can be used instead of linear regression. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Training steps.}
The parameters $T$ and $\boldsymbol{\upbeta}$ are trained using the following steps:
\begin{enumerate}
    \item Perform LLM inference using a prompt once for each instance of the training set. Record the token probabilities for each instance of the training set.
    \item Select a temperature $T$ that maximizes the linear correlation of $\textrm{score}_{\textrm{softmax}}^{(T)}$ (computed from the token probabilities using \autoref{eq:score-softmax}) with the gold scores of the training set.
    \item Fit a linear regression model $\boldsymbol{\upbeta}^\mathsf{T}\mathbf{x}$ on the training set.\Rev*{A5}{ (The clipping in \autoref{eq:g-scale} is applied only at inference time.)}
\end{enumerate}

The temperature scaling and linear regression, which differentiate our method from \textsc{G-Eval}, are not specific to LCP and can be applied to any tasks where the fit of continuous values is important. \textsc{G-Eval} \cite{liu-etal-2023-g} was originally proposed for ranking alternative LLM outputs in NLG tasks, where linear correlation and fit are not important, as long as the score has a good rank correlation with human judgment.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Prompt for LCP.}
We employ a few-shot prompt with simple instructions (see \appref{sec:lcp-prompt}) and the complexity scale description used for the MultiLS English dataset \cite{shardlow-etal-2024-extensible}. Similar to our LS prompt, we try to avoid difficult words in the prompt itself (e.g., ``complexity''), and explicitly target language learners. We instruct the model to rate the difficulty for an ``intermediate language learner'' to obtain consistent complexity ratings. The prompt is not based on \textsc{G-Eval} prompt methodology \cite{liu-etal-2023-g} and does not elicit chain-of-thought output. We did not perform preliminary experiments with multiple prompts; instead, we performed an ablation of the prompt, as detailed in \autoref{sec:experiments-ls}. 

\textsc{G-Eval} involves a specific methodology for creating a prompt. The fixed step-by-step instructions in the prompt are referred to as ``chain-of-thoughts'' (CoT) by \textsc{G-Eval}'s authors \cite{liu-etal-2023-g}. This is an unusual usage of the term, as ``CoT prompting'' has been used to refer to the model \emph{generating} a description of thoughts leading to a conclusion in response to the prompt \cite[also cited by Liu et al.]{wei_etal_2022_chain}. In the context of LCP methods based on \textsc{G-Eval}, ``CoT'' was implemented both as \textsc{G-Eval}-style fixed instructions \cite{enomoto-etal-2024-tmu} and as per-instance thought process generation \cite{smadu-etal-2024-investigating}. Both result in an increased computational cost: the former involves a longer prompt (see \appref{sec:lcp-prompt-tmu-hit}) compared to ours and the latter results only in a small improvement \cite{smadu-etal-2024-investigating} while incurring a longer generated output compared to a single token in our case. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experiments}

For the experiments, we used GPT-4o\footnote{\texttt{gpt-4o-2024-08-06}} \cite{openai_etal_2024_gpt4o}, a paid closed-weight model, with the following parameters: \texttt{frequency\_penalty=0.5} and \texttt{presence\_penalty=0.3}. For our methods, we used \texttt{temperature=0}, and when performing stochastic inference, we used \texttt{temperature=0.7}. For LCP, we additionally used \texttt{logprobs=True} and \texttt{top\_logprobs=20} to obtain token log-proba\-bil\-ities, which are necessary for \textsc{G-Scale} and \textsc{G-Eval}.
For a fair comparison, the choice of penalties and temperature for stochastic inference followed previous methods \cite{aumiller-gertz-2022-unihd,enomoto-etal-2024-tmu}, which also used OpenAI's GPT models.

For LCP using \textsc{G-Scale}, we selected the temperature $T = 2.0$, which achieved the highest mean correlation on the trial sets of all languages from values (0.1, 0.2, \ldots, 1.9, 2.0). The parameters $\boldsymbol{\upbeta}$ were fitted separately using the trial set for each language using linear regression with L2 regularization ($\alpha = 1$).

Unless specified otherwise, we used the prompt templates described in Appendices \ref{sec:ls-prompt} and  \ref{sec:lcp-prompt} for LS and LCP, respectively. For LCP, we used 3-shot examples sampled from the trial data for each language such that their scores are as close as possible to 1, 5, and 3 (the lowest point, highest point, and midpoint of the scale, respectively), and we rounded their scores to the nearest integer. The same three examples for each language were used at all times.

We performed evaluation on the MultiLS datasets for ten languages (including \MLSJ). We reported results in each language and the mean of the results computed across languages. The best results for each language and for the mean are printed in \textbf{bold}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 6
\begin{table}[b]
\input{04table06.tex}
\hangcaption{Accuracy@1@top1 of LS on the MultiLS datasets compared with the top MLSP shared task submission results (ISEP, GMU, TMU-HIT) reported in the shared task findings \protect\cite{shardlow-etal-2024-bea}.}
\label{tab:ls-acc1top1}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 7
\begin{table}[b]
\input{04table07.tex}
\hangcaption{Accuracy (equal to potential@1 and MAP@1) of LS on the MultiLS datasets compared with the top MLSP shared task submission results (ISEP, GMU, TMU-HIT) reported in the shared task findings \protect\cite{shardlow-etal-2024-bea}.}
\label{tab:ls-acc}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Lexical Simplification}
\label{sec:experiments-ls}

We compared the performance of our system with the top MLSP shared task submissions \cite{shardlow-etal-2024-bea}, 
namely ISEP \cite{dutilleul-etal-2024-isep}, GMU \cite{goswami-etal-2024-gmu}, and TMU-HIT \cite{enomoto-etal-2024-tmu} using accuracy@1@top1 and accuracy in Tables \ref{tab:ls-acc1top1} and \ref{tab:ls-acc}, respectively. Accuracy@1@top1, which was the main evaluation metric of the shared task, strictly evaluates the top generated simplification against the top simplification given by human annotators, whereas accuracy evaluates the top generated simplification against all simplifications given by annotators, counting less frequent simplifications as correct too. \Rev*{A6b}{To enable comparison with the top shared task submission, we used the shared task's evaluation script to compute the metrics. As a result, the maximum achievable accuracy and accuracy@1@top1 for languages with unsimplifiable instances was lower than 1 \cite[Sec.~4.2, App.~A]{shardlow-etal-2024-bea}. For Filipino, the maximum accuracy was 0.772; for other languages, it was in the interval [0.997, 1].}

Our system outperforms the top shared task submissions on most languages, achieving the best performance on 7 out of 10 languages in both accuracy@1@top1 and accuracy, thereby setting a new state-of-the-art result on the MultiLS datasets.

The top shared task submission, TMU-HIT used GPT-4, whereas we used GPT-4o, a newer model by OpenAI, which may be more effective in this task, although its API usage is cheaper. Because our system and TMU-HIT use different underlying models, prompts, and inference methods, we ran another set of experiments, where we used the same model (GPT-4o) and  the following prompt/inference method combinations to perform further analysis:
\begin{itemize}
    \item \textbf{Ours / Median Rank:} Using our prompt, we repeat the inference 10 times with temperature 0.3, similarly to TMU-HIT, but instead of ensembling the simplifications using the mean rank, we ensemble them using the median rank.
    \item \textbf{Ours / TMU-HIT:} Using our prompt, we repeat inference 10 times with temperature 0.3, and ensemble the simplifications using the mean rank, exactly as TMU-HIT did.
    \item \textbf{TMU-HIT / Ours:} Using the TMU-HIT prompt,\footnote{We use the TMU-HIT prompt for languages other than Japanese (see \appref{sec:ls-prompt-tmu-hit}). We do not evaluate TMU-HIT's method for the Japanese language, which requires specific postprocessing.} we greedily decode output, as outlined in \autoref{sec:method-ls}.
    \item \textbf{Ours / Ours:} Using our prompt, we greedily decode output.
\end{itemize}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 8
\begin{table}[b]
\input{04table08.tex}
\hangcaption{Number of inferences per instance (\#Inf.), accuracy@1@top1 and accuracy of LS on the \linebreak MultiLS English, Japanese, and Spanish datasets by prompt and inference method. Significance levels: $p < 0.001$ ($^{\ast\ast\ast}$), $p < 0.01$ ($^{\ast\ast}$), $p < 0.05$ ($^{\ast}$).}
\label{tab:ls-acc-prompt-inference}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

To evaluate the results, as shown in \autoref{tab:ls-acc-prompt-inference}, we used the same 10 stochastic inferences for the first two rows of the table, and a single greedy inference for each of the last two rows to ensure comparability. We computed $p$-values to indicate whether the difference from the best performing system in each column is statistically significant. To this end, we used the paired permutation test with 20,000 resamples of the dataset instances. 

We can observe that using the TMU-HIT prompt instead of ours consistently resulted in poorer performance. The system using the TMU-HIT prompt achieved significantly lower accuracy@1@top1 in Japanese and significantly lower accuracy in Japanese and Spanish.

None of the other results differ significantly. However, an important difference lies in the number of inferences used:
Ensembling 10 stochastic inferences with the same prompt using mean rank (TMU-HIT) or median rank yields results without any significant difference ($p \ge 0.05$) from performing a single greedy inference (our method). 

In a manual analysis of the outputs, we observed a non-intuitive behavior of averaging ranks using the arithmetic mean: an item that has the top rank in the majority of cases may not obtain the top mean rank if it is weighed down by low ranks (or omission of the item) in the other cases. If ranks are averaged using the median instead, an item that has the top rank in the majority of cases will always get the top median rank. The effect on accuracy metrics, however, was mixed---while median rank performed better for Japanese and Spanish, it performed worse for English---and in none of the cases was the effect significant.

We can conclude that our prompt outperformed the prompt used by TMU-HIT \cite{enomoto-etal-2024-tmu}, and more importantly, our inference method achieved essentially the same results at one tenth of the computational cost of TMU-HIT's inference method. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 9
\begin{table}[b]
\input{04table09.tex}
\hangcaption{Analysis of LS errors in terms of accuracy (instances where the top system simplification is not among gold simplifications by human annotators) of our system on \MLSJ.}
\label{tab:error-analysis}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Error analysis on \MLSJ}
\label{sec:error-analysis}

On the \MLSJ dataset, our method achieved an accuracy of 0.721. In other words, in 72.1\% of the test set instances, the top predicted simplification was in the gold data. We manually analyzed the errors in the remaining 27.9\% (159 instances), categorizing them by issue in \autoref{tab:error-analysis} (examples in Appendix \ref{sec:error-examples}). In 21\% of the errors, we found the simplifications valid, although they did not match the gold data. For the actually invalid simplifications, most errors stemmed from the meaning of the sentence not being preserved.


While the amount of meaning lost or changed varies, the inability to consistently preserve meaning would be a serious problem for a real-world application: a system making grammatical errors or occasionally producing relatively complex outputs may still be usable, but a system that does not preserve meaning would not be fit for deployment. We therefore believe that meaning preservation should be prioritized in future LS research. The general formulation of LS as a task, which does not involve deletion or summarization, makes it well-suited for settings where meaning preservation is crucial.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 10
\begin{table}[b]
\input{04table10.tex}
\hangcaption{$R^2$ of lexical complexity prediction on the MultiLS datasets, compared with the top MLSP shared task submission and frequency baseline results reported in the shared task findings \protect\cite{shardlow-etal-2024-bea}, and ablated versions of the \textsc{G-Scale} scoring function. In addition to printing the best result in \textbf{bold}, we \underline{underlined} the second best result.}
\label{tab:lcp-main}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Lexical Complexity Prediction}

In \autoref{tab:lcp-main}, we compare the performance of our system with the top MLSP shared task \cite{shardlow-etal-2024-bea} submissions Archaeology \cite{cristea-nisioi-2024-archaeology}, GMU \cite{goswami-etal-2024-gmu}, TMU-HIT \cite{enomoto-etal-2024-tmu}, and the shared task's frequency baseline using the coefficient of determination $R^2$. We use $R^2$, which is computed as a linear function of the mean squared error,\footnote{This is consistent with the shared task evaluation \cite{shardlow-etal-2024-bea} and the \texttt{r2\_score} implementation in scikit-learn (\url{https://scikit-learn.org/}) but different from $R^2$ defined merely as a square of the Pearson's correlation coefficient.} as it is the strictest evaluation metric. A more lenient evaluation using Pearson's and Spearman's correlation coefficients can be found in Appendix \ref{sec:lcp-corr}. Our method, \textsc{G-Scale}, outperformed all shared task submissions in 9 of 10 languages, and in the mean over languages, it outperformed them by a wide margin. It also outperformed all ablated versions of our scoring function (also shown in \autoref{tab:lcp-main}). When evaluated using correlation coefficients (in Appendix \ref{sec:lcp-corr}), the differences between methods were smaller, because errors in mean and variance (and linear correlation, in the case of the Spearman's rank coefficient) were not penalized. Yet, our method still achieved better mean correlation than the top shared task systems and ablations.

The ablations of the scoring function were:
\begin{itemize}
\item \textbf{Direct:} We predicted the score represented by the highest probability token (greedy decoding of a single token)
\item \textbf{Direct + LR:} We used the value represented by the highest probability token as an independent variable for linear regression.
\item \textbf{\textsc{G-Eval}}: We used the token probabilities to compute $\textrm{score}_\text{\textsc{G-Eval}}$ (\autoref{eq:score-geval}).
\item \textbf{\textsc{G-Eval} + LR:}   We used the value of $\textrm{score}_\text{\textsc{G-Eval}}$ as an independent variable for linear regression.
\item \textbf{\textsc{G-Scale} -- LR:}  We calibrated $T$ and use the token probabilities to compute $\textrm{score}_\text{softmax}^{(T)}$ (\autoref{eq:score-softmax}).
\end{itemize}
For fairness of comparison, we always clipped the values predicted by linear regression to the range [0, 1], in the same way it is done for \textsc{G-Scale}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 11
\begin{table}[b]
\input{04table11.tex}
\hangcaption{$R^2$ of lexical complexity prediction on the MultiLS English, Japanese, and Spanish datasets, compared with the prompt ablations (described in \appref{sec:ls-prompt}) and the prompt used by TMU-HIT with a minor modification (see \appref{sec:ls-prompt}). Our scoring function (\textsc{G-Scale}) is used in all cases.}
\label{tab:lcp-prompt}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

In \autoref{tab:lcp-prompt}, we ablated the prompt and compared it with a prompt adapted to our method from the top MLSP shared task submission, TMU-HIT. To save computational resources, we evaluate only on three languages (English, Japanese, and Spanish). Our prompt achieved the best mean performance, the best performance in English and Japanese, and the second best performance in Spanish. This also demonstrates that assigning a concrete role in the prompt, i.e., ``You are an intermediate learner of \emph{\{language\}}'', was beneficial even if it did not match the actual target population of the datasets: Japanese was annotated by rather advanced non-native speakers, English by 10 natives and 11 non-native university students, and Spanish was annotated mostly by native speakers. With \textsc{G-Scale}, it was not important whether the scale of scores output by the LLM was shifted, as long as it was consistent. The concrete role seems to have helped the LLM achieve this consistency. Additionally, specifying the language at the end of the prompt, i.e., ``assign a difficulty rating to the \emph{\{language\}} word'' was beneficial too, except for Spanish, where it slightly hurt the performance. Note that we used the same 3-shot examples in all the compared prompts.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 12
\begin{table}[b]
\input{04table12.tex}
\hangcaption{Corpus-based features: $R^2$ of lexical complexity prediction on the MultiLS English, Japanese, and Spanish datasets. We use frequency from wordfreq and TUBELEX, and range (number of YouTube channels in which a word occurs) from TUBELEX. Features that outperform LLM-only prediction are \textit{italicized}.}
\label{tab:lcp-features}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We aimed to further improve the performance on English, Japanese, and Spanish by adding corpus-based features to \textsc{G-Scale}, namely log-frequency and log-range. As shown in \autoref{tab:lcp-features}, the best mean performance is achieved by adding log-range from the TUBELEX corpus \cite{nohejl-etal-2025-film}. Range is a lexical dispersion measure, whose logarithm correlates particularly well with lexical complexity in the MultiLS datasets and psycholinguistic variables \cite{nohejl_watanabe_2025_dispersion}. The logarithm of word frequency is the most common feature used for lexical complexity prediction. Both log-frequency and log-range alone (upper half of the table) provided very competitive predictions compared to \textsc{G-Scale} based on LLM prompting, but the best performance was achieved by their combination. This shows that the LLM predictions and log-frequency (or log-range) were relatively uncorrelated, complementing each other in LCP. In Appendix \ref{sec:wordfreq} we evaluated the addition of log-frequency to all ten languages. For three of them, it hurt the performance, which we assume reflects the low quality of the respective corpora.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}

We built the first LS dataset targeting non-native Japanese speakers, \MLSJ. Among Japanese LS datasets, the proposed dataset has the unique feature of  providing gold labels not only for LS but also for the subtask of LCP. \MLSJ is based on diverse source texts, has a permissive license (CC-BY-SA), and is part of the larger family of ten language-specific MultiLS datasets.

We proposed and evaluated methods for LS and LCP that are applicable to any language supported by an LLM. In the case of LCP, our method can be calibrated on a small number of examples (we used the trial set of 30 examples from each language-specific MultiLS dataset) and it can leverage corpus-based features in addition to LLM prompting. The computational cost of both methods is an order of magnitude smaller than that of the previous state-of-the-art LLM-based approaches.

For LS, we used a simple prompting approach, which nonetheless outperformed the state-of-the-art results for most languages. Our approach consists of a prompt that led to this performance improvement and an inference method that reduces the computational cost to one tenth of the previous state of the art.

We proposed \textsc{G-Scale}, a novel method for rating text input on a continuous scale using LLMs. \textsc{G-Scale} outperformed the previous state-of-the-art LLM-based LCP method \cite{enomoto-etal-2024-tmu}, which is based on \textsc{G-Eval} \cite{liu-etal-2023-g}, as well as a previous feature-based approach \cite{cristea-nisioi-2024-archaeology}. \textsc{G-Scale} can efficiently combine other features with LLM prompting and leverage training examples while keeping the computational cost at a single LLM inference per data instance. Through ablation, we demonstrated that both differences of our method from \textsc{G-Eval} result in improved performance on LCP. \Rev*{comm-B2c}{We plan to apply \textsc{G-Scale} to other rating tasks in future work.}

We believe that together with the Japanese LS and LCP dataset, our LS and LCP methods, which are widely applicable across languages, form a solid basis for future research into language-specific or personalized methods.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%\phantomsection
\section*{Limitations}
\label{sec:limitations}

\MLSJ was carefully constructed as an evaluation dataset. Its main limitation, shared with the other currently available MultiLS datasets, is that it provides only 30 training examples in the trial set. We attempted to provide  resources for the evaluation of personalization, with lexical complexity ratings and demographic profiles for ten individuals. The lack of data for personalization is a weak point shared by all LS and LCP datasets and an important area for future research.


\Rev*{B3c}{\MLSJ targets non-native speakers of Japanese, holders of JLPT levels N2 or N1 (or the older equivalents 2 and 1) whose first language is not Chinese or Korean. While lexical complexity annotation was performed by individuals from this population, lexical substitutions were provided by professionals: as explained in \autoref{sec:annotation}, the individuals from the target population would not be able to perform the simplification of words that are difficult to understand for them. We strove to find professionals capable of performing the task and to provide them with appropriate instructions. Nonetheless, our approach does not guarantee optimal substitutions for the target population. We suggest exploring annotation methods that would involve feedback from individuals in the target population for future research.}

\Rev*{comm-B1b}{Uncommon \emph{kanji}, namely characters outside the top 2,500 \emph{kanji} in Japanese Wikipedia, do not occur in the MultiLS-Japanese dataset. In Japanese Wikipedia, which has a relatively high proportion of both proper names and specialized vocabulary, such characters are used to spell only 0.32\% of the words in running text}.\footnote{The top 50 of the excluded words by frequency include Japanese and Chinese proper names, e.g., 齋藤 (\emph{Sait\={o}}), 馮 (\emph{F\'{e}ng}); words that have a more common alternative spelling, e.g., 蛋白質 (\emph{protein}), commonly spelled タンパク質; and words regularly spelled using the uncommon characters, e.g., 嫉妬 (\emph{jealousy}), 弾劾 (\emph{impeachment}).
} Similarly, words with frequency less than $10^{-7.5}$ in TUBELEX, corresponding to 0.24\% tokens in TUBELEX, were excluded.\footnote{Many of such words are proper nouns or English words in \emph{katakana} and the alphabet. Simplifiable Japanese words in the top 50 excluded words by frequency include 多党 (\emph{multiparty}) or 綬章 (\emph{a medal of honor with a ribbon}).} Given the limited size of our dataset (600 target words), we preferred to represent words that are difficult for the target population, yet relatively frequent. This may be limiting in the future as LS systems improve: it will become desirable for evaluation datasets to focus more on lower-frequency words, which are potentially also more difficult to simplify. As for lower frequency \emph{kanji}, a distinction can be made between lexical complexity proper and complexity stemming from rare \emph{kanji}, which is commonly handled in Japanese by adding \emph{furigana} (a phonetic guide in the form of ruby characters, e.g., the small characters かいらい in \ruby[g]{傀儡}{かいらい}政権, \emph{a puppet regime}). MultiLS-Japanese does not contain \emph{furigana}; we have not specifically tested annotators for \emph{kanji} proficiency or experimented with the effect of adding \emph{furigana} on perceived complexity or comprehension.




In both tasks, we used the same method across datasets for all ten languages, avoiding prompt or feature engineering for individual datasets. We did this to demonstrate the general applicability of our methods; however, at the same time, it is a limitation of our approach. We expect that further performance gains can be achieved by considering language specifics, target population, or the text genre, and adjusting the prompts and features used by our methods accordingly.

We evaluated \textsc{G-Scale}, our method for rating text input on a continuous scale using LLMs, only on LCP. Its performance on other tasks is yet to be investigated. Both of our methods were devised to be applicable to closed-weight LLMs available only via API, such as OpenAI's GPT model family, relying only on prompting and access to token probabilities. In principle, our methods are applicable to any LLM, but we have not evaluated them on other models.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%\phantomsection
\section*{Ethical Considerations}
\label{sec:ethical}

We manually reviewed the content and language of the \MLSJ dataset, both original sentences and their simplifications, and found no ethically problematic content. Nevertheless, some instances of colloquial language may be considered inappropriate depending on the context. Discretion is therefore advised when using the dataset for purposes beyond its intended use (evaluating LCP and LS systems), for instance, in a classroom setting.

The following target words occurring in the dataset are informal and may be inappropriate in certain contexts, though none of them is particularly offensive: 奴 (\emph{fella}, see \autoref{tab:ex-selected} for context), ブッとばす (\emph{send flying}, \emph{knock down}, also in \autoref{tab:ex-selected}), おまえ (\emph{you}, informal), 愚かな (\emph{silly, stupid}), ケチる (\emph{to be stingy, to skimp}, informal), キレさせる (\emph{make} sb.\ \emph{lose their temper}, \emph{make} sb.\ \emph{snap}, informal). Other informal words appear in the sentence contexts and sometimes in simplifications. Such words are also listed in existing pedagogical vocabulary resources: やつ, おまえ, \mbox{愚か}, \mbox{けち} \cite{sunakawa_etal_2012_construction}, and ぶっ飛ばす \cite{ninjal_2025_compound} are variants of the words above. While not typical examples of textbook vocabulary, we believe that such words are an important part of the complex lexicon that LS for non-native speakers should be tested against.

Text simplification, including LS, involves a trade-off between simplicity and meaning preservation \cite{agrawal-carpuat-2024-text}. We have highlighted meaning preservation as a priority for future research in \autoref{sec:error-analysis}. However, several issues beyond the preservation of factual meaning are worth noting. Compared to translation, it is more difficult for simplification to preserve the register or negative and positive nuances. For example, in our dataset, the simplifications of the formal and neutral word 犯人 (\emph{criminal, culprit}) include 悪い人 (\emph{bad person}) and 悪い奴 (\emph{bad guy}, informal), which could be perceived as more negative, and simplifications of ケチる (\emph{to skimp}, informal), which carries a negative connotation, include 節約する (\emph{to save}, \emph{to economize}), introducing a more positive tone. Similar issues were identified in the Catalan MultiLS dataset \cite{saggion-etal-2024-lexical}. We therefore recommend that text simplification systems, particularly when intended to assist vulnerable individuals, are deployed only with human oversight. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment

The \MLSJ dataset was created as part of a joint research project between Nara Institute of Science and Technology and Nikkei, Inc. We are grateful to the anonymous reviewers for their insightful comments and suggestions.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.8}
\bibliography{04refs}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\appendix
\vspace{-1\Cvs}

\section{English--Japanese Glossary and Manual Dataset Tags}
\label{sec:glossary}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table Appendix01
\begin{table}[b]
\vspace{-2\Cvs}
\input{04tableZ01.tex}
\vspace{0.5\Cvs}
\sTag{} marks abbreviations that are also used as manual tags in the dataset (see Appendix \ref{sec:format}).\\
\tblfn{1}
For convenience, we understand as auxiliaries (AUX) all of the following: auxiliary verbs, including copulas (e.g., だ), attached typically to AN; the so-called ``conjunctive particles'' (接続助詞) that attach to verbs (e.g., て, ば); the light verb する used in honorific verb templates or with attributive forms AN, ADV (e.g., はっきりした); the dummy verb いる (after て); and the dummy copula ある (in である).\\
\tblfn{2} In the dataset, we categorize the so-called ``conjunctive particles'' (接続助詞) that attach to verbs (e.g., て, ば) in the convenience auxiliary category (AUX).
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\clearpage

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{The MultiLS-Japanese Dataset Format}
\label{sec:format}

The dataset is available in a TSV (tab-separated values) format, where the fields below are columns and data instances are rows. See Appendix \ref{sec:glossary} for POS tags used for manual tagging.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table Appendix02
\begin{table}[b]
\vspace{-2\Cvs}
\input{04tableZ02.tex}
\end{center}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\clearpage

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Guidelines for Determining Target Word Spans}
\label{sec:guidelines}

The \textbf{Guidelines} we used to determine the boundaries of each of the 600 target words in the \MLSJ dataset are divided into Appendices \ref{sec:g-cd},  \ref{sec:g-dep}, and \ref{sec:g-mwe}, going bottom--up in terms of linguistic structure. In each section's \textbf{Results}, we report how the respective guidelines are reflected in targets' manual tags, and how many targets they were applied to. See \autoref{sec:morpho} for our motivation and an overview of basic segmentation units (SUW, LUW, and \emph{bunsetsu}).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Compounding and Derivation}
\label{sec:g-cd}

In Japanese, compounding and derivation can be very productive and the resulting words effectively constitute Sino-Japanese phrases (e.g., ``962名収容可能'' ``able to accommodate 962 persons''). These words would typically constitute a single LUW. In many cases, it is possible to simplify their constituents independently, targeting units smaller than words from the perspective of Japanese morphology.

Note that two-morpheme native Japanese (\emph{wago}) compounds, of which verb+verb compound verbs are a prominent example, and some words that could be considered Sino-Japanese  (\emph{kango}) compounds are segmented as a single SUW, and therefore do not require special handling.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Guidelines:}
\begin{enumerate}[label=\arabic*.,leftmargin=2em]
\item If a word is productively formed by derivation or compounding and the target part can be simplified independently, include only the necessary SUWs of the word. e.g., the prefix 翌 in ``翌1988年8月29日'' or the two-SUW compound 収容可能 in a longer compound ``962名収容可能'' can be easily simplified independently.
\item Otherwise, include all constituents (SUWs) of a word, in particular:
\begin{enumerate}[label=\alph*.,ref=\theenumi\alph*,leftmargin=2em]
    \item prefixes, including honorific, e.g., \underline{ほの}暗い, \underline{お}菓子,
    \item suffixes, e.g., 配偶\underline{者}, 何\underline{ら},
    \item compound constituents, e.g., \shortul{チケット}\shortul{販売}\shortul{委託}\shortul{会社},
    \item\label{item:cpart} all constituents of a complex particle, e.g.\shortul{に}\shortul{つい}\shortul{て}, \shortul{に}\shortul{つき}\shortul{まし}\shortul{て},
    \item all constituents of a complex auxiliary, e.g., \shortul{ざる}\shortul{を}\shortul{得}\shortul{ない}, and the verb it attaches to.
\end{enumerate}
\end{enumerate}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Results:}
We manually tagged the morphology of the target words in the field \texttt{main\_morphology}. Target words that are part of a larger whole have a non-empty field \texttt{whole}. See Appendix \ref{sec:format} for details. There are 30 such target words: 29 nouns and 1 adjectival noun.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Dependent Words (\emph{Fuzokugo})}
\label{sec:g-dep}

A sequence of dependent words (SUWs) may be included in a target word, typically spanning at most one \emph{bunsetsu}. In many cases, these dependent words (SUWs) can be thought of as inflections. Note, however, that inflections not analyzed as separate SUWs do not need special handling, e.g., ない \textrightarrow{} な\underline{けれ}, and we therefore do not discuss them.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Guidelines:}
\begin{enumerate}[label=\arabic*.,leftmargin=2em,resume]
\item Always include the following dependent words (SUWs):
\begin{enumerate}[label=\alph*.,ref=\theenumi\alph*,leftmargin=2em]
    \item the light verb する, e.g., はっきり\underline{する}, がっかり\underline{する},
    \item auxiliary verbs attached to verbs and adjectives, e.g., なす\underline{べき}, 麗しかっ\underline{た},
    \item\label{item:te1} the conjunctive particle て/で in the \emph{te}-form of verbs and adjectives, e.g., 浮ん\underline{で}, 麗しく\underline{て}, and the dummy verb いる attached to the \emph{te}-form if expressing a state, e.g., 酔っ払って\underline{いる},
    \item copula forms (inflections) attached to adjectival nouns, e.g., 巨大\underline{な}, \underline{で}, 頻繁\underline{に},
    \item particles attached to adverbs, e.g., ぽつん\underline{と},
    \item particles and copula forms attached to attributively (adverbially) used nouns, e.g., ぼろぼろ\underline{の}, ぼろぼろ\underline{だ}, 常\underline{に},
    \item sequences of the above words, e.g., 取り上げ\shortul{られ}\shortul{た} (aux.\ られる + aux.\ た).
\end{enumerate}

\item Do not include the following dependent words (SUWs), except if necessary for simplification:
\begin{enumerate}[label=\alph*.,ref=\theenumi\alph*,leftmargin=2em]
    \item\label{item:part} particles other than those mentioned above: among others, case particles, e.g., が, から,
    \item\label{item:te2} anything following the \emph{te}-form, except the dummy verb いる expressing a state,
    \item syntactically compounding (suffix-like) verbs attached to the adverbal form (連用形), e.g., 忘れる, 得る, すぎる, にくい
    \item dependent words preceding the independent word, e.g., \Midline{を}連れて unless they form a complex particle e.g., \underline{を}めぐって or a similar fixed expressions, e.g., \underline{に}ならい.
\end{enumerate}

\item When in doubt about dependent words, think of the possible simplifications and include them if necessary to allow those simplifications.
\end{enumerate}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Results:}
You can find the dependent parts of targets in the fields \texttt{aux} and \texttt{dep}, see Appendix \ref{sec:format}. 201 of the targets have at least one.

Note that the so-called ``conjunctive particle'' (接続助詞) て/で, used pervasively in the \emph{te}-form of verbs and adjectives, requires special handling (see \ref{item:te1} and \ref{item:te2} above). We generally do include other conjunctive particles in targets (e.g., ば in 手に入れなけれ\Midline{ば}, see \ref{item:part}), except when they are part of complex auxiliaries (e.g., ば in なけれ\underline{ば}ならない, or つつ in \mbox{\underline{つつ}も}, see \ref{item:cpart}).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Multiword Expressions}
\label{sec:g-mwe}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Guidelines:}

\begin{enumerate}[label=\arabic*.,leftmargin=2em,resume]
\item If an expression (collocation) can be easily simplified independently, do not include unnecessary words, e.g., 愚痴 in the collocation 愚痴を言う (to complain, lit.\ ``to say a complaint'') can be simplified independently.
\item Otherwise, include the minimal expression that can be simplified easily, e.g., all words of the idiom 愚痴をこぼす (to complain, lit.\ ``to spill a complaint'') need to be included.
\end{enumerate}

\paragraph{Results:} All MWE target words in the dataset have \texttt{main\_pos} value ``MWE'', see Appendix \ref{sec:format}. The dataset contains the following MWEs: 

\begin{itemize}[label=--,leftmargin=2em]
    \item 36 noun-verb (N-V): e.g., 責任を持つ (to bear responsibility)
    \item 2 noun phrase (NP), e.g., 川の氾濫 (river flooding),
    \item 1 noun-adjective (N-ADJ): 体格のいい (to have a good physique),
    \item 1 verb-verb (V-V): 打って変わって (completely change to).
    \item 1 adverb-verb (ADV-V): ぱっとせん (to be inconspicuous),
    \item 1 particle-verb (PART-V): にならい (following the example of),
\end{itemize}

\clearpage

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Annotation Instructions and Profile Questionnaires}
\label{sec:inst}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Lexical Complexity Annotation Instructions}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 2
\begin{figure}[b]
\begin{center}
\includegraphics{32-4ia4f02.eps}
\end{center}
\caption{Lexical complexity instructions, page 1 of 2: instructions in Japanese.}
\label{fig:lcp1}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 3
\begin{figure}[t]
\begin{center}
\includegraphics{32-4ia4f03.eps}
\end{center}
\caption{Lexical complexity instructions, page 2 of 2: instructions in English.}
\label{fig:lcp2}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\clearpage


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Lexical Complexity Profile Questionnaire}
\label{sec:lcaq}
\begin{enumerate}
\item		日本語能力試験 (JLPT) レベル。過去に合格したレベルのうち、最も高いものをメニューから選んでください。	\\
		Your JLPT level. Please choose from the menu the highest level you hold a JLPT certificate for.\\
        \textbf{Options:} N5, N4, N3, N2, N1, (2009以前) 4, (2009以前) 3, (2009以前) 2, (2009以前) 1

\item		年齢。	\\
		Your age.	\\
        例: 「25」
			
\item		小学校から数えて、あなたが学校に行った年数。	\\
		How many years did you go to school (spent in formal education) counting from elementary school?	\\
        例: 「16」(小学校6年、中学校3年、高校3年、 大学4年のとき、6+3+3+4→16 / elementary 8y, high school 4y, university 4y: 8+4+4→16)
			
\item		母語。いくつかある場合は、コンマ（、）で区切り、記入してください。	\\
		Your native language. If you have multiple native languages, please separate them with commas.	\\
        例: 「タイ語」「Tagalog, English」
			
\item		母語を除いて話せる言語の数。	\\
		How many languages can you speak, except your native language(s)?\\
        例: 「1」(母語と日本語だけの場合 / if you speak only your native language and Japanese)
			
\item		1週間で日本語を読む時間数。	\\
		How many hours a week do you read in Japanese?\\	例: 「3」「0.5」
			
\item		大学や大学の「日本語予備教育」で日本語を勉強したことがありますか。	\\
		Have you studied Japanese at university, or in ``Japanese language preparatory education'' for university?\\
        \textbf{Options:} はい, いいえ
			
\item		今、日本に住んでいますか。	\\
		Do you currently live in Japan?
        \textbf{Options:} はい, いいえ
			
\item		合わせて日本に住んでいた年数。	\\
		How many years have you lived in Japan in total?	\\ 例: 「0」「0.5」「3」

\end{enumerate}

\clearpage

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Lexical Simplification Annotation Instructions}
\label{sec:inst-ls}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 4
\begin{figure}[b]
\vspace*{-5\Cvs}
\begin{center}
\includegraphics{32-4ia4f04.eps}
\end{center}
\caption{LS instructions, page 1 of 2: initial instructions.}
\label{fig:ls1}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\clearpage

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 5
\begin{figure}[t]
\begin{center}
\includegraphics{32-4ia4f05.eps}
\end{center}
\hangcaption{LS instructions, page 2 of 2: additional examples given after reviewing the trial set annotation (first 10 sentences).}
\label{fig:ls2}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{English Translation}
The following translation of the text shown in Figures \ref{fig:ls1} and  \ref{fig:ls2} was not given to the annotators, who were all native Japanese speakers.

\medskip
\noindent
\textbf{1. Fill in information about yourself}

\noindent
Please, fill in information about yourself in the ``Profile'' sheet.

\medskip
\noindent
\textbf{2. Annotation}

\noindent
Annotation work is performed in the ``Annotation'' sheet. The ``Annotation'' sheet contains 200 sentences to be annotated. Each sentence contains exactly three underlined words or phrases (targets).
The task is to list simple paraphrases (simplifications) that can be substituted for these targets. The following conditions must be met:

\begin{itemize}
\item The simplification preserves most of the meaning of the target in the original sentence.
\item The simplification is easier for learners of Japanese (do not assume they are native speakers of Chinese or Korean) to understand than the target.
\item When the target is replaced by the simplification, the sentence is grammatically correct and the meaning is understandable.
\end{itemize}

\noindent
List at most three simplifications for each target. Below the sentence, there is a table for simplifications of each target. Please fill in the simplification starting from the leftmost column. If you can't think of any, just enter the target as it is in the field ``Simplification (1)''.

\noindent
You may refer to dictionaries and other resources while working, but please do not use ChatGPT or other automated tools. Please list simplifications that you consider appropriate, based on your knowledge and experience in Japanese language teaching.

\medskip
\noindent
\parbox{\textwidth}{\emph{[Image of the annotation interface, with fields for each target word labeled ``Simplification (1--3)'']}}\hspace{-1em}
\medskip

\noindent
When listing simplifications, please pay attention to the following points:

\begin{itemize}
\item If the difficulty of the target is due to word forms or functional expressions, feel free to change them:\\
○ ought to do → should do\quad ○ know not → don't know \emph{[English approximations]}

\item However, assume that the terminal form (dictionary form) and the adverbal form + \emph{masu} (\emph{masu}-form), or the adverbal form and the adverbal form + \emph{te} (\emph{te}-form) are all easy to understand word forms, and do not change one to the other.

\emph{[Examples of unnecessary changes of verb form]}

\item The POS and the number of words in the target and the simplification can be different if they fit the context.\\
○ bottomless → without end \emph{[English approximation]}
\item Simply changing the target's kanji is to hiragana or katakana is not a simplification.\\
× 望ましい → のぞましい \emph{[Change to an easier spelling]}
\item Simplifications should use the most common orthography:\\
× ほんとう \emph{[Easier but less common]} \quad  ○ 本当\\
× 所為  \emph{[More difficult and also less common]} \quad ○ せい
    
\item Simplification should not be supplemented with readings.\\
× 躊躇する → 躊躇(ちゅうちょ)する \emph{[Reading (ruby) supplied in parentheses]}
\end{itemize}

\medskip
\noindent
\textbf{Examples based on the first part of the annotation}

\noindent
\emph{[Summary:} The Japanese text uses actual good and bad examples of substitutions from the first part of annotation to illustrate the following points:

\begin{enumerate}
\item Prefer rough equivalents over accurate but long explanations, which make the resulting sentence more difficult to comprehend.

\item Make sure the substitutions are grammatical.

\item Make sure the substitutions are easier for language learners than the original word.

\item Do not change the verb form unless necessary for the simplification.

\item Do not suppose that the learners understand English. Only use loanwords to the extent they are natural in Japanese.\emph{]}
\end{enumerate}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\clearpage
\subsection{Lexical Simplification Profile Questionnaire}
\label{sec:lsaq}

The questions 1, 2, 3, 4 in the following questionnaire are identical to questions 2, 3, 5, 6 given to lexical complexity annotators (see \appref{sec:lcaq}). All of the LS annotators were native Japanese speakers living in Japan.

\begin{enumerate}

\item		年齢。　例: 「25」
			
\item		小学校から数えて、あなたが学校に行った年数。　 例: 「16」\\(小学校6年、中学校3年、高校3年、 大学4年のとき、6+3+3+4→16)
			
\item		母語を除いて話せる言語の数。　 例: 「1」
			
\item		1週間のうち日本語を読む時間数。　例: 「3」「0.5」
			
\item		日本語教育の経験年数。　 例: 「4」
			
\item		1週間のうち日本語を教える時間数。　例: 「1」

\end{enumerate}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection*{English Translation}

The following translation of the text above was not given to the annotators, who were all native Japanese speakers.
\begin{enumerate}
\item		Your age.	
\item		How many years did you go to school (spent in formal education), counting from elementary school?
\item		How many languages can you speak, except your native language(s)?
\item		How many hours a week do you read in Japanese?
\item		How many years of experience teaching Japanese as a second language do you have?
\item		How many hours a week do you teach Japanese?	
\end{enumerate}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Lexical Simplification Corrections}
\label{sec:correction}

The authors corrected the LS annotations, making  134 manual edits, among which 45 edits were made in the trial set (0.150 edits per instance and annotator) and 89 in the test set (0.016 edits per instance and annotator), reflecting that the implemented quality control measures (manual check, individual feedback, instruction clarification using additional examples) were effective.

The corrections are outlined below:

\begin{enumerate}
\item Perform a conventional character normalization. We automatically normalize the substitutions to Unicode NFKC, except for the full-width tilde character (～), which is almost exclusively used in the full-width form in Japanese.
\item Ensure simplifications are different from targets. (a) If the simplification is just an orthographic variant of the target or its reading in hiragana, we delete it, e.g., 強力な: 剛力な, もの: 者. (b) In ambiguous cases, we do not delete the simplification, e.g., 凸凹 (reading でこぼこ or とつおう): でこぼこ. (c) If the simplification differs from the target just by lacking a very common productive prefix or suffix while keeping the stem, we delete it, e.g., 男っぽ過ぎる: 男っぽい, お菓子: 菓子.
\item Ensure the simplifications preserve the original word form. Simplifications should not unnecessarily change the word form, in particular, the basic level of formality or politeness. In cases they do, we adjust the word form.
\item Correct grammatical errors. Simplifications should fit the context grammatically. (a) If a simplification can be corrected by a simple change of word form, we correct it, e.g., 死んだ \textrightarrow{} 死んで. (b) If a simplification repeats a word (typically a particle or a light verb) that is already contained in the context, we remove the repeated word, e.g., 家で \textrightarrow{} 家 (で already follows the target). (c) If a simplification misses a word (typically a particle or a light verb) that is part of the target, we add that word, e.g., 発表 \textrightarrow{} 発表されます (されます was in the target). (d) If more substantial changes are necessary (addition of a word not present in the target, replacement of a word, or a POS conversion), we delete the simplification. In particular, we do not make any changes that would require a choice between several possible edits.
\item Perform an orthographic normalization. Simplifications should use the most common spelling of each word. If the simplification uses a less common spelling, we correct it, e.g., 阿呆 \textrightarrow{} アホ.
\item Check simplicity. Simplifications should be easier to understand than the target words. In general, we do not examine simplification for complexity or frequency. If, however, a simplification has any of the issues above, and in addition to that, contains words much rarer or difficult to comprehend than the target, we delete it.
\item Check semantics. Simplifications should preserve meaning. In general, we do not examine simplification for preservation of meaning. If, however, during the formal checks, we discover a simplification that clearly does not preserve the core meaning, we delete it, e.g., an affirmative instead of a negative, or 美容師 ``beautician'' simplified as 技術者 ``technician''.
\end{enumerate}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Prompt Templates}

In the following prompt templates, only the $\dlsh$ symbols represent line breaks. Italic text in curly braces (e.g., \texttt{\textit{\{word\}}}) denotes placeholders to be replaced when constructing the prompt.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Our Lexical Simplification Prompt}
\label{sec:ls-prompt}

For English, the placeholder \texttt{\textit{\{language\}}} and the subsequent space in the following prompt template are omitted; for other languages, it is replaced with the name of the language (e.g., ``Japanese''):
\begin{itemize}\item[]
\begin{raggedright}%
\noindent%
\ttfamily\small%
Answer with a list of ten different, simpler \textit{\{language\}} words Y to replace the difficult word X in the given sentence S. Each of the simpler words Y must follow these rules:\ttnl
1.\ When we replace the difficult word X with the simpler word Y, the sentence S is grammatically correct.\ttnl
2.\ When we replace the difficult word X with the simpler word Y, the sentence S has the same meaning as before.\ttnl
3.\ The simpler word Y is easier to understand than the difficult word X for language learners.\ttnl
\#\#\# Sentence S: \textit{\{context\}}\ttnl
\#\#\# Difficult word X: \textit{\{word\}}\ttnl
\#\#\# List of simpler words Y:\\
\end{raggedright}
\end{itemize}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{TMU-HIT Lexical Simplification Prompt}
\label{sec:ls-prompt-tmu-hit}

We run a separate experiment using exactly the same prompt template as that used for languages other than Japanese by the TMU-HIT system \cite{enomoto-etal-2024-tmu}. For English, the placeholder \texttt{\textit{\{language\}}} and the subsequent space in the following prompt template are omitted; for other languages, it is replaced with the name of the language (e.g., ``Japanese''):
\begin{itemize}\item[]
\begin{raggedright}%
\noindent%
\ttfamily\small%
I will give you a \textit{\{language\}} sentence and a word in the \textquotesingle{}Sentence\textquotesingle{} and \textquotesingle{}Word\textquotesingle{} format.\ List ten alternatives for the Word that are easier to understand, separated by \textquotesingle{},\textquotesingle{}.\ \ttnl
You must follow these four rules.\ttnl
1.\ Take into account the meaning of the Word in the Sentence.\ttnl
2.\ Alternatives must be easier to understand than the Word.\ttnl
3.\ Each alternative consists of one word.\ttnl
4.\ Do not generate an explanation.\ttnl
\ttnl
Sentence:\ \textit{\{context\}}\ttnl
Word:\ \textit{\{word\}}\ttnl
Alternatives:\\
\end{raggedright}
\end{itemize}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Our Lexical Complexity Prediction Prompt}
\label{sec:lcp-prompt}

The placeholder \textit{\{language\}} in the following prompt template is replaced with the name of the language (e.g., ``Japanese'', ``English''), and the placeholder \textit{\{examples\}} is replaced with a concatenation of $n$ few-shot examples in the following format:

\begin{itemize}\item[]
\begin{raggedright}%
\noindent%
\ttfamily\small%
Sentence:\ \textit{\{context$_i$\}}\ttnl
\ttnl
Word:\ \textit{\{word$_i$\}}\ttnl
\ttnl
Difficulty rating:\ \textit{\{score$_i$\}}\ttnl
\ttnl%
\end{raggedright}
\end{itemize}
The prompt template:
\begin{itemize}\item[]
\begin{raggedright}%
\noindent%
\ttfamily\small%
\underline{You are an intermediate learner of \textit{\{language\}}.}\textsubscript{\textnormal{[role]}}\ You will be given a sentence and a word included in the sentence.\ Your task is to assign a rating to the word based on how difficult to understand the word is for you.\ttnl
\ttnl
Based on the following list, assign a difficulty rating from 1 to 5 to the word.\ttnl
   - 1: Very Easy - Words which are very familiar to you\ttnl
   - 2: Easy - Words which are mostly familiar to you\ttnl
   - 3: Neutral - When the word is neither difficult or easy\ttnl
   - 4: Difficult - Words which you are unclear of the meaning, but may be able to infer from the context\ttnl
   - 5: Very Difficult - Words that you have never seen before, or are very unclear\ttnl
\ttnl
\underline{Please assign a difficulty rating to the \textit{\{language\}} word.}\textsubscript{\textnormal{[final]}}\ttnl
\ttnl
\textit{\{examples\}}Sentence:\ \textit{\{context\}}\ttnl
\ttnl
Word:\ \textit{\{word\}}\ttnl
\ttnl
Difficulty rating:\\
\end{raggedright}
\end{itemize}

\bigskip
In addition to this prompt, we experiment with the following ablations:

\begin{enumerate}
    \item \textbf{No language:} The underlined sentence marked as [final] is replaced with:\\\texttt{\small Please assign a difficulty rating to the word.}
    \item \textbf{No language, generic role:} The combination of (1) and replacing the underlined sentence marked as [role] with:\\ \texttt{\small You are an intermediate language learner.}
    \item \textbf{No language, no role:} The combination of (1) and (4).
    \item \textbf{No role:} The underlined sentence marked as [role] is removed.
\end{enumerate}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{TMU-HIT-like Lexical Complexity Prediction Prompt}
\label{sec:lcp-prompt-tmu-hit}

We run a separate experiment using a prompt template based on the prompt $P_\textrm{role}$ used by the TMU-HIT system \cite{enomoto-etal-2024-tmu} for English and Portuguese. We modify their prompt only by replacing the original scores represented by multiple tokens (0.0, 0.25, 0.5, 0.75, 1.0) with single-token ones (1, 2, 3, 4, 5), so that it can be used in our \textsc{G-Scale} method.

The placeholder \textit{\{examples\}} is replaced with a concatenation of $n$ few-shot examples in the following format:

\begin{itemize}\item[]
\begin{raggedright}%
\noindent%
\ttfamily\footnotesize%
Sentence:\ \textit{\{context$_i$\}}\ttnl
\ttnl
Word:\ \textit{\{word$_i$\}}\ttnl
\ttnl
Complexity:\ \textit{\{score$_i$\}}\ttnl
\ttnl%
\end{raggedright}
\end{itemize}
The prompt template:
\smallskip
\begin{itemize}\item[]
\begin{raggedright}%
\noindent%
\ttfamily\footnotesize%
You are an individual without specialized knowledge or expertise in a specific area.\ttnl
\ttnl
You will be given a sentence and a word included in the sentence.\ttnl
\ttnl
Your task is to rate the word on one metric.\ttnl
\ttnl
Please make sure you read and understand these instructions carefully.\ Please keep this document open while reviewing, and refer to it as needed.\ttnl
\ttnl
\ttnl
Evaluation Criteria:\ttnl
\ttnl
Complexity (1, 2, 3, 4, 5):\ the complexity of a word in terms of how difficult the word is to understand.\ttnl
\ttnl
Evaluation steps:\ttnl
1. Read the sentence and word carefully to understand the context.\ttnl
2. Determine the complexity of the word based on the following criteria:\ttnl
\ \ \ - 1:\ The word is simple and easily understandable to most people.\ttnl
\ \ \ - 2:\ The word may have some complexity or be specific to a certain field, but can still be understood with some effort.\ttnl
\ \ \ - 3:\ The word is moderately complex and may require some background knowledge or explanation to understand fully.\ttnl
\ \ \ - 4:\ The word is quite complex and may be difficult to understand without significant knowledge or explanation.\ttnl
\ \ \ - 5:\ The word is extremely complex and likely only understood by experts or individuals with specialized knowledge.\ttnl
3. Assign a complexity rating to the word.\ttnl
\ttnl
Note:\ Your own familiarity with a word should not impact your rating.\ This should be based on an average person \textquotesingle{}s understanding of the word.\ttnl
\ttnl
\textit{\{examples\}}Sentence:\  \textquotesingle\textit{\{context\}}\textquotesingle\ttnl
\ttnl
Word:\  \textquotesingle\textit{\{word\}}\textquotesingle\ttnl
\ttnl
Complexity:%
\end{raggedright}
\end{itemize}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\clearpage
\section{Evaluation of LCP Using Correlation}
\label{sec:lcp-corr}

\autoref{tab:lcp-main-r} and \autoref{tab:lcp-main-rho} evaluate the same systems as \autoref{tab:lcp-main}, but using Pearson's and Spearman's correlation coefficients, respectively, instead of $R^2$. As both correlation coefficients are invariant to the application of a linear function with a positive slope, ablations of linear regression are omitted.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 13
\begin{table}[h]
\input{04table13.tex}
\hangcaption{Pearson's correlation coefficient of lexical complexity prediction on the MultiLS datasets, compared with the top MLSP shared task submission and frequency baseline results reported in the shared task findings \protect\cite{shardlow-etal-2024-bea}, and ablated versions of the \textsc{G-Scale} scoring function.}
\label{tab:lcp-main-r}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 14
\begin{table}[h]
\input{04table14.tex}
\hangcaption{Spearman's rank correlation coefficient of lexical complexity prediction on the MultiLS datasets, compared with the top MLSP shared task submission and frequency baseline results reported in the shared task findings \protect\cite{shardlow-etal-2024-bea}, and ablated versions of the \textsc{G-Scale} scoring function.}\label{tab:lcp-main-rho}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\clearpage

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Wordfreq Log-Frequency as a Feature}
\label{sec:wordfreq}

In \autoref{tab:lcp-wf}, we compare the performance of linear regression based on log-frequency only, \textsc{G-Scale} without additional variables and \textsc{G-Scale} with log-frequency as an additional variable. For log-frequency, we use \texttt{wordfreq}, a Python library \cite{robyn_speer_2022_wordfreq} combining frequency data from multiple corpora, e.g.\ Wikipedia, Twitter, and film subtitle corpora. For Sinhala, \texttt{wordfreq} does not provide any data. Therefore, we substitute a word frequency list representative of diverse domains \footnote{\url{https://github.com/nlpcuom/Word-Frequency-List-for-Sinhala}} \cite{fernando-dias-2021-building}. The MLSP shared task's baseline was also based mostly on \texttt{wordfreq}.

The mean $R^2$ drops when we add log-frequency as a feature, but this drop is actually driven by only three languages: Catalan, Filipino, and Sinhala. For all other languages, the addition of log-frequency results in an improvement. For Catalan, Filipino, and Sinhala, the log-frequency alone has a negative $R^2$ and hence the negative effect is not surprising. As all three are low-resource languages, we hypothesize that this reflects the limited quality of the \texttt{wordfreq} data for them. Without further analysis, however, we also cannot rule out issues with annotation quality or word distribution of the trial sets of these languages.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 15
\begin{table}[h]
\input{04table15.tex}
\hangcaption{Using \texttt{wordfreq} log-frequency as an additional variable: $R^2$ of lexical complexity prediction on the MultiLS datasets.}
\label{tab:lcp-wf}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\clearpage

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Examples of Lexical Simplification Errors}
\label{sec:error-examples}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 16
\begin{table}[h]
\input{04table16.tex}
\hangcaption{Examples of LS errors in terms of accuracy (instances where the top system simplification is not among gold simplifications by human annotators) of our system on \MLSJ. See analysis in \autoref{tab:error-analysis}.}
\label{tab:my_label}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\clearpage

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}%

\bioauthor[:]{Adam Nohejl}{%
Adam Nohejl received his bachelor's and master's degrees in computer science from Charles University in 2009 and 2011, respectively, and a bachelor's degree in Japanese studies from Charles University in 2018. He worked as an independent software developer. He is currently a Ph.D.\ student at the Nara Institute of Science and Technology. His research interests include applications of natural language processing to text simplification, readability, and language learning.
}
\bioauthor[:]{Akio Hayakawa}{%
Akio Hayakawa received his bachelor's and master's degrees in education from the University of Tokyo in 2015 and 2017, respectively, and obtained a master's degree in engineering from the Nara Institute of Science and Technology in 2023. He is currently a Ph.D.\ student at the Universitat Pompeu Fabra. His research focuses on applying natural language processing to education and social inclusion.
}

\bioauthor[:]{Yusuke Ide}{%
Yusuke Ide received his B.A.\ degree from the University of Tokyo in 2017 and a master's degree in engineering from the Nara Institute of Science and Technology in 2023.
He is currently a Ph.D.\ student at the Nara Institute of Science and Technology.
His research interests lie in natural language processing, particularly lexical semantics and readability.
}

\bioauthor[:]{Taro Watanabe}{
Taro Watanabe received his B.E.\ and M.E.\ degrees in information science from Kyoto University in 1994 and 1997, respectively, and obtained an M.S.\ degree in Language and Information Technologies from the School of Computer Science, Carnegie Mellon University in 2000.
In 2004, he received a Ph.D.\ in informatics from Kyoto University.
After working as a researcher at ATR, NTT and NICT, and as a software engineer at Google, he is a professor at the Nara Institute of Science and Technology starting in 2020.
His research interests include natural language processing, machine learning, language modeling and machine translation.
}
\end{biography}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\biodate




\clearpage














\clearpage















































\end{document}


