\documentclass[english]{jnlp_3.8}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{array}
\usepackage{hangcaption_jnlp}
\allowdisplaybreaks

\makeatletter
\let\MYcaption\@makecaption
\makeatother

\usepackage{subcaption}
\captionsetup{compatibility=false}

\makeatletter
\let\@makecaption\MYcaption
\makeatother
\setlength{\captionwidth}{\textwidth}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition
\usepackage{soul}
%%%%\usepackage{url}
%%%%\usepackage{amsthm}
\usepackage{booktabs}
\usepackage{algorithm}
\usepackage{multirow}
\usepackage{xspace}
%%%%\usepackage{array}
%%%%\urlstyle{same}
%%%%\usepackage{tikz}
%%%%\usepackage{comment}
%%%%\usepackage{footnote}
%%%%\usepackage{subcaption}
%%%%\usepackage{pgfplots}
\newcommand{\rteenagers}{\subreddit{teenagers}}% \newcommand{\citet}[1]{\citeauthor{#1} \shortcite{#1}}
%%%%\newcolumntype{P}[1]{>{\centering\arraybackslash}p{#1}}
\newcommand{\ex}[1]{\textit{#1}\xspace}
\newcommand{\gl}[1]{``#1''\xspace}
\DeclareMathOperator{\softmax}{softmax}
\DeclareMathOperator{\ReLU}{ReLU}
%%%%\newcommand{\tabref}[1]{Table~\ref{#1}\xspace}
    \newcommand{\tabref}[1]{Table \ref{#1}\xspace}
%%%%\newcommand{\figref}[1]{Figure~\ref{#1}\xspace}
    \newcommand{\figref}[1]{Figure \ref{#1}\xspace}
%%%%\newcommand{\eqnref}[1]{Equation~(\ref{#1})\xspace}
    \newcommand{\eqnref}[1]{Equation (\ref{#1})\xspace}
\newcommand{\method}[2][]{\textsc{#2}$_{\text{#1}}$\xspace}
\newcommand{\wordtovec}{\method{word2vec}}
\newcommand{\skipgram}{\method{Skip-gram}}
\newcommand{\cbow}{\method{CBOW}}
\newcommand{\BERT}[1][]{\method[#1]{BERT}}
%%%%\newcommand{\contexttovec}{\method{context2vec}}
\newcommand{\PreTrained}[2][]{\textnormal{#2}$_{\text{#1}}$\xspace}
%%%%\newcommand{\RoBERTa}[1][]{\PreTrained[#1]{RoBERTa}}
\newcommand{\task}[1]{\textit{#1}\xspace}
\newcommand{\contexttotarget}{\task{context2target}}
\newcommand{\targettocontext}{\task{target2context}}
\newcommand{\subreddit}[1]{\texttt{r/#1}\xspace}
\newcommand{\rireland}{\subreddit{ireland}}
\newcommand{\raustralia}{\subreddit{australia}}
%%%%\newcommand{\rleagueoflegends}{\subreddit{leagueoflegends}}
%%%%\newcommand{\rmusic}{\subreddit{Music}}
%%%%\newcommand{\rsports}{\subreddit{sports}}
\newcommand{\rchan}{\subreddit{4chan}}
\newcommand{\rhiphopheads}{\subreddit{hiphopheads}}
%%%%\newcommand{\rprogramming}{\subreddit{programming}}
%%%%\definecolor{myblue}{RGB}{0, 114, 178}
%%%%\definecolor{myorange}{RGB}{255, 165, 0}
%%%%\definecolor{mygreen}{RGB}{153, 200, 102}
%%%%\definecolor{mypurple}{RGB}{102, 0, 153}
%%%%\definecolor{mybrown}{RGB}{121, 67, 33}
\newcommand{\review}[3][black]{%
  {#3}%
}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{32}
\Number{1}
\Month{March}
\Year{2025}
%%%%\Issuetitle{Special Issue: XXXXXXXX}

\received{2024}{8}{26}
\revised{2024}{10}{21}
\accepted{2024}{11}{27}

\setcounter{page}{134}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Discovering Unusual Word Usages with Masked Language Model via Pseudo-label Training}
\eauthor{Tatsuya Aoki\affiref{titech} \and Jey Han Lau\affiref{unimelb} \and Hidetaka Kamigaito\affiref{naist} \and Hiroya Takamura\affiref{titech} \and \\
Timothy Baldwin\affiref{unimelb}\affiref{mbzuai} \and Manabu Okumura\affiref{titech}}
\eabstract{
User-generated texts contain not only non-standard words such as \ex{b4} for \ex{before}, but unusual word usages such as \ex{catfish} for a person who uses fake identity online, which requires knowledge about the words to handle such cases in natural language processing.
We present a neural model for detecting the non-standard usages in social media text.
To deal with the lack of training data for this task, we propose a method for synthetically generating pseudo non-standard examples from a corpus, which enables us to train the model without manually-annotated training data and for any arbitrary language.
Experimental results on Twitter and Reddit datasets show that our proposed method achieves better performance than existing methods, and is effective across different languages.
}
\ekeywords{Non-standard Word Usage, Masked Language Model, Pseudo Data}

\headauthor{Aoki et al.}
\headtitle{Discovering Unusual Word Usages with MLM via Pseudo-label Training}

\affilabel{titech}{}{Institute of Science Tokyo}
\affilabel{unimelb}{}{The University of Melbourne}
\affilabel{naist}{}{Nara Institute of Science and Technology (NAIST)}
\affilabel{mbzuai}{}{MBZUAI}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

\renewcommand{\thefootnote}{}
\footnote[0]{This research was conducted by the first author while a Ph.D. candidate at Tokyo Institute of Technology (renamed Institute of Science Tokyo in 2024), independent of any employer. The findings and views expressed are solely those of the authors.}
\renewcommand{\thefootnote}{\arabic{footnote}}
\vspace{-2\Cvs}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}

Social media text often contains slang and other non-standard language usages, posing problems for natural language processing (NLP) \cite{eisenstein-2013-bad}.
Much research has addressed this issue at the word-type level, where a word-type refers to a unique word within a text corpus, tackling expressions such as \ex{b4} to mean \ex{before} or \ex{bitter} to mean \ex{bitterly cold} \cite{aw-etal-2006-phrase,han-baldwin-2011-lexical,han-2013,chen-2015,barteld-2017-detecting,stewart-eisenstein-2018-making,kulkarni-wang-2018-simple,Lourentzou-2019,cho-kim-2021-google,sun-etal-2022-semantically}.
However, to fully understand non-standard language use, it is essential to also examine word-token level usages, where a word-token refers to a specific instance of a word as it appears in a particular context. For example, consider the word \ex{catfish} in the sentence \ex{The person turned out to be a catfish from an online dating app}, which means \gl{a person who uses a fake online profile}. In such examples, when a word exists in the dictionary but is used in a way that diverges from its standard definition, we refer to this as non-standard word usage.\footnote{We follow \citeB{aoki2019} with the definition of nonstandard. This is different from the work done by \citeB{stewart-eisenstein-2018-making} where they discard the words such as ``catfish'' which can be found in a newspaper as they mainly focus on rare or ill-formed words such as ``b4''.}

Non-standard word usages present a challenge for NLP systems, as they require an understanding of context and meaning that extends beyond traditional word-type level processing. Such creative usages are often employed to communicate sensitive or harmful content while evading censorship\footnote{Acknowledging the potential risk of dual use, which could inadvertently aid censors in identifying and preventing such usages.} \cite{huang-etal-2013-resolving,zhang-etal-2015-context,steen_you_2023}, necessitating systematic detection to prevent online harms \cite{fillies-paschke-2024-simple}.
Non-standard usages also affect downstream NLP tasks; for instance, Google Translate struggles to accurately interpret terms like \ex{catfish} when translating into languages such as Japanese.
However, relatively few researchers address this issue, leaving limited resources available for studying non-standard usages.
\review[myblue]{MAB-1-2, AP-2-1}{For instance, \citeA{sun-etal-2024-toward} constructed a slang dataset from English movie subtitles, and \citeA{aoki-etal-2017-distinguishing} developed a dataset of non-standard usages based on Japanese Twitter posts.
Despite ongoing efforts, there remains a lack of training datasets that cover a wide range of domains, which often results in semi-supervised methods struggling to transfer knowledge across domains \cite{ruder-plank-2018-strong}.}
This limitation highlights the necessity of unsupervised approaches to more effectively detect and handle these non-standard usages.
Moreover, as researchers highlight that non-standard word usages are not confined to English but occur in other languages as well \cite{sboev-2016-sources,aoki-etal-2017-distinguishing}, it is essential to develop a scalable, language-independent approach for acquiring and understanding non-standard word usages.

\review[myblue]{MAB-4}{This paper aims to discover non-standard word usages to gain a deeper understanding of informal language use without relying on dedicated language resources. We propose a language-independent method for detecting non-standard word usages from large-scale text streams, such as Reddit, and introduce a new dataset of non-standard word usages to support research in this field.
\review[myblue]{MAB-1-2}{Unlike \citeA{sun-etal-2024-toward}, who focused on movie subtitles, we concentrate on online communities because studying authentic, user-generated text is essential for capturing the dynamic and evolving nature of non-standard expressions in real-world contexts.}
In our approach, we hypothesize that by first addressing the word usage classification task, in which the goal is to classify whether a word usage in a sentence is standard or non-standard, we can then apply this classification to all potential words to detect non-standard usages. Figure \ref{fig:example_task} presents an example of input and output for the word usage classification task, as predicted by our proposed model. As Figure \ref{fig:example_task} shows, an input sentence and a target word are provided in this classification task. In the context of Figure \ref{fig:example_task}, ``nova'' is used in a non-standard way that refers a rank or skill level in gaming, and the target word ``nova'' is successfully classified as non-standard by our work.}
Our proposed method is a simple but effective approach for detecting non-standard word usages in context. 
To deal with the absence of labeled data, we train our model over pseudo training data, eliminating the need for labeled datasets.
Unlike previous models such as \citeA{aoki-etal-2017-distinguishing,aoki2019}, which utilize local fixed-size context, our model takes into account broader context \review[mygreen]{AS-4-2}{through a masked language model \cite{devlin-etal-2019-bert,roberta}, allowing for the detection of non-standard usages based on richer contextual signals.}
\review[mygreen]{AS-3-1}{Although recent research utilizes large language model (LLM)-based methods for similar tasks \cite{fillies-paschke-2024-simple,sun-etal-2024-toward}, we focus on a lightweight masked language model-based method, as applying LLM-based methods is both financially and computationally expensive when attempting to discover non-standard usages from massive text streams like Twitter or Reddit.}
\review[myblue]{MAB-4}{In the experiment, we first evaluate the word usage classification task using a Japanese Twitter dataset, which is an annotated corpus for word usage classification. We then extend our method to non-annotated English Reddit dataset to discover non-standard usages, with the evaluation conducted by crowd-workers.} The experimental results on Japanese Twitter dataset and English Reddit dataset demonstrate that our proposed model is able to detect non-standard usages from social media across different languages.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 1 (table?)
\begin{figure}[t]
\begin{center}
\begin{tabular}{l}
\toprule
Model Input \\
\midrule
\textbf{Sentence}: \emph{If you were actually good enough to be in low \underline{\textbf{nova}}, then you wouldn't be in silver.} \\
\textbf{Target Word}: \textit{nova} \\
\midrule
\midrule
\textbf{Model Output}: \textit{non-standard} \\
\bottomrule
\end{tabular}
\end{center}
\caption{Example input and output for the word usage classification task.}
\label{fig:example_task}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Our  contributions are as follows: (1) we propose a novel approach to detecting non-standard word usages, which we show to outperform existing methods over multiple languages; and (2) we develop a large-scale dataset of non-standard usages of English words.
The dataset is publicly available.\footnote{\url{https://github.com/tatsuokun/reddit_slang_usage_dataset}}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Methodology}

\review[myblue]{MAB-4}{This paper addresses a word usage classification task, where we are given an input sentence and a target word within the sentence. The goal is to determine whether the usage of the target word is standard or non-standard as Figure \ref{fig:example_task} illustrates.}
\review[myorange]{BS-1}{We assume that for non-standard usages, both (1) predicting the target word from its surrounding context and (2) predicting the context words from the target word present challenges. However, we hypothesize that these prediction tasks provide important clues for detecting non-standard usages. Based on this insight, we propose a model that incorporates vector representations produced from both \contexttotarget prediction task, where we predict a target word from its surrounding context words, and \targettocontext prediction task, where we predict the surrounding context words from the target word. By combining the information from these two approaches, our model seeks to enhance the accuracy of identifying non-standard word usages.}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Background}
In this section, we briefly review methods for  \contexttotarget prediction and \targettocontext prediction commonly used in NLP.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{\contexttotarget prediction}
\label{subsec:context2target}

Masked language models such as \BERT \cite{devlin-etal-2019-bert} or RoBERTa \cite{roberta} are trained to predict masked tokens from a given context.\footnote{\cbow \cite{sgns} is an earlier well-known method for learning word embeddings based on \contexttotarget prediction.
We include experiments based on CBOW in this paper, but find that masked language model performs better than \cbow.}
A masked language model takes as input a masked sentence such as $X' = \{w_1, w_2, [MASK], w_4, ..,w_n\}$ (where the original token $w_3$ is masked as $[MASK]$), and predicts the ground-truth masked token $w_3$ from a contextual representation.
\review[mygreen]{AS-4-3}{The masked language model utilizes bi-directional transformers for capturing contextual representation, which are composed of multi-headed self-attention mechanisms and element-wise feedforward networks \cite{transformers}.}
\review[mygreen]{AS-4-3, AS-4-4}{On top of these transformer layers, both BERT and RoBERTa build a prediction layer known as language modeling head (LM head),\footnote{We follow an implementation by HuggingFace. \url{https://github.com/huggingface/transformers/blob/main/src/transformers/models/roberta/modeling_roberta.py#L1116}} which is an architecture designed to predict masked tokens through a vocabulary-sized projection layer. This LM head is trained using the cross-entropy loss function, where the model learns to predict masked tokens during training. Combined with the softmax function, this layer generates a probability distribution over all possible tokens, allowing the model to predict the masked token by selecting the one with the highest probability.} The LM head is implemented by first applying a fully-connected layer to the input features coming from bi-directional transformers, followed by the GELU activation function \cite{gelu} and layer normalization \cite{layernorm}.
\review[mygreen]{AS-4-4}{After that}, the processed features are projected back to the size of the vocabulary through a projection layer that includes a bias term. This architecture plays an important role as a masked language model, enabling the prediction of masked tokens from the given context.

Since the masked language model directly models target word prediction based on context, it can be used to \review[mypurple]{AP-3-1}{capture difference of} training contexts for a given target word.
Thus, in our model, we use contextualized representations obtained from a masked language model as a feature for \contexttotarget prediction.In the experiment, we used the hidden representation obtained from the LM head by excluding the final projection layer, specifically utilizing the output after applying the fully-connected linear layer, GELU activation, and layer normalization, and finetune for the classification task.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{\targettocontext prediction}

\skipgram \cite{sgns} is a method for learning word embeddings based on predicting context words from a target word in vector space, by assigning two different vector representations to every word in the vocabulary: one for that word as a target word, and one as a context word.
Given a target word in context, \skipgram is trained by maximizing the dot-product between the vector for the target word and vectors for the context words.
We regard this training procedure as \targettocontext prediction, where we assume that the vectors trained by \targettocontext prediction will produce a different \review[mygreen]{AS-4-5}{aspect} to the vectors obtained from \linebreak
\contexttotarget prediction.
As proposed by \citeA{aoki2019}, this training procedure can also model how a target word fits in context by manipulating the trained vector representations, suggesting that \targettocontext prediction is helpful in classifying word usages.
Based on their finding that those two different representations from \skipgram called IN and OUT embeddings play an important role in distinguishing word usages, we also incorporate these two distinct vectors derived from \skipgram into our model.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Word Usage Classifier}

\review[myblue]{MAB-4}{In this work, we classify a target word in a sentence as either standard or non-standard using a binary word classifier which takes into account of contextual features from \targettocontext prediction and \contexttotarget prediction.
For the task formulation, given} a sentence with $n$ words $X = \{w_1,..,w_n\}$ and a target word $w_{\text{target}} \in X$, we classify $w_{\text{target}}$ as $y  \in \{0, 1\}$, where $0$ indicates a standard usage and $1$ indicates a non-standard usage.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 2
\begin{figure}[b]
\begin{center}
\includegraphics{32-1ia6f2.eps}
\end{center}
\caption{Overview of our proposed model.}
\label{figure:overview} 
\vspace{-0.5\Cvs}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\figref{figure:overview} presents an overview of our proposed method.
We first feed sentence $X$ into the masked language modeling layer and 
 get ${\bf o}_{\text{target}}$ as a \contexttotarget representation of 
$w_{\text{target}}$.\footnote{Each word is tokenised into wordpieces to reduce 
the vocabulary size.}
 We use the mean of the contextualized embeddings for each of the 
wordpiece tokens that make up the target word. Note that we keep the 
target word unmasked for the training of non-standard word detection, (i.e. we do not replace the target word $w_{\text{target}}$ with $[MASK]$), as 
we find that masking the original word negatively impacts on results, 
consistent with the findings of \citeA{zhou-etal-2019-bert} in the 
context of lexical substitution.
We also incorporate a \targettocontext representation ${\bf e}_{\text{target}}$, in the form of a word embedding of the target word $w_{\text{target}}$ learnt by \skipgram.
We concatenate the two vectors to form ${\bf z} = [{\bf o}_{\text{target}};\ {\bf e}_{\text{target}}]$, as the combined representation of the target word.
Finally, we feed ${\bf z}$ into a 2-layered multi perceptron with ReLU activation \cite{relu} and softmax function to implement binary classification, as follows:
\begin{equation}    
    {p (\hat{\bf y}| {\bf z})} = {\softmax}({\bf W}_2\, {\ReLU} ({\bf W}_1{\bf z} + {\bf b}_1)+{\bf b}_2),
\label{eq:classification}
\end{equation}
where {${\bf W}_1$}, {${\bf W}_2$}, {${\bf b}_1$}, {${\bf b}_2$} are weight matrices and bias vectors. 
To train the classifier, we use binary cross entropy loss:
\begin{equation} 
    L(y, \hat{y})= - \frac{1}{D} \sum^{D}_{i=0} y_i  {\rm log}(\hat{y}_i) + (1-y_i)  {\rm log}(1-\hat{y}_i),
\label{eq:loss}
\end{equation}
where $y_{i} \in {\{0, 1\}}$ is the gold label, $\hat{y_{i}}$ is the model prediction (i.e.\ probability of non-standard usage), and $D$ is the number of examples in the minibatch.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Pseudo Non-Standard Examples}

\eqnref{eq:loss} requires the gold label $y$ to train the model, meaning that we need a large-scale source of annotated data.
However, constructing a large-scale dataset is expensive as it needs expert lexicographic knowledge. 
To counter the lack of manually-annotated training data for this task, we incorporate the idea of pseudo-example learning \cite{bergsma-etal-2008-discriminative,poon-etal-2009-unsupervised,pauls-klein-2012-large,kiyono-etal-2019-empirical}, where we automatically generate a dataset of synthetic non-standard examples from a raw corpus, and use it to train the model.
That means that our model does not need any annotated training data and can be trivially adapted to different domains and languages, given that we can easily generate training data given a monolingual corpus.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 3
\begin{figure}[t]
\begin{center}
\includegraphics{32-1ia6f3.eps}
\end{center}
\caption{Pseudo-example creation.}
\label{figure:dataset}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\figref{figure:dataset} depicts the method for constructing the training dataset with pseudo examples.
The procedure is composed of two steps: (i) choose a target word in a sentential context as a standard example; and (ii) replace the selected word with an alternative word based on uniform random sampling from vocabulary in the training corpus, and regard the modified sentence as a non-standard example.
\review[mypurple]{AP-1-2}{Note that we iterate the target word selection over all possible nouns in each sentence, meaning that the same sentence is repeatedly used during training based on the number of nouns it contains.}
Since this approach does not require any human annotation, we can train the model on any language, and present experiments on both English and Japanese.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experiments}

We evaluate our method on two different datasets which are manually annotated for non-standard usages in context: (1) a pre-existing Japanese Twitter dataset \cite{aoki2019}; and (2) an English Reddit dataset which is novel to this research.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Word Usage Classification Task}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Experimental Settings}

\citeA{aoki2019} built a Japanese Twitter dataset for word usage classification, where a target word in each sentence is manually annotated as to whether its usage in context makes sense relative to common-sense expectations.
\review[mypurple]{AP-1-1}{Following their research, we focus on non-standard usages of noun cases.}
We use a subset of this dataset in our experiments, after filtering out word segmentation errors in the dataset.\footnote{Some target words in the dataset are not used as nouns but as a part of a named entity.}
To perform the word segmentation filtering, we used two Japanese morphological analyzers---Mecab \cite{kudo-etal-2004-applying} and Juman++ \cite{tolmachev-etal-2018-juman}---and selected those target words which both analyzers identified as a single-morpheme common noun.
\review[mygreen]{AS-2-1}{The filtering process eliminated 385 sentences from the original dataset, resulting in a total of 2,588 sentences, consisting of 1,380 standard usages and 1,208 non-standard usages.}
Note that this Twitter dataset is only used for evaluation and not for training, fine-tuning, or hyper-parameter tuning.
To train our model, we took the BCCWJ corpus \cite{maekawa-etal-2010-design} and extracted one million (standard) examples from the corpus, from which we generated one million pseudo-examples according to the process described in \figref{figure:dataset}.
For hyper-parameter tuning, we prepared a separate validation dataset from BCCWJ made up of 1,000 sentences evenly split between standard and pseudo non-standard examples.

As evaluation metrics, we use accuracy and average precision to measure overall classification performance, and precision, recall, and F1 score to assess detection performance specifically for non-standard usages.
We report averaged performances across three different random seeds.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Model Details}

Following \citeA{aoki2019}, we learn word embeddings via \skipgram with negative sampling over the BCCWJ corpus \cite{maekawa-etal-2010-design}, with dimensionality 300, and the number of negative samples set to 10.
We also incorporate IN and OUT embeddings produced from \skipgram forming \targettocontext representation in \eqnref{eq:classification} as ${\bf e}_{\text{target}} = [{\bf v}^{\text{IN}}_{\text{target}};\ {\bf v}^{\text{OUT}}_{\text{target}}]$ where ${\bf v}^{\text{IN}}_{\text{target}}$ and ${\bf v}^{\text{OUT}}_{\text{target}}$ are the IN and OUT embedding for the target word.
In regard to this point, we provide an ablation study on feature representation for a target word.
For the masked language model, we use pre-trained Japanese \PreTrained[base]{RoBERTa},\footnote{\url{https://huggingface.co/nlp-waseda/roberta-base-japanese}}$^{,}$\footnote{\label{disclaimer-base-large}Due to computational resources and limitations, we use the base model instead of the large model.} which was trained over Japanese Wikipedia and the Japanese portion of CC-100.
\review[mygreen]{AS-2-2}{During training, we freeze the pre-trained weights of the word embeddings and bi-directional transformers in RoBERTa, and only finetune a part of LM head described in Section \ref{subsec:context2target} and the word usage classification layer.}
Other training details are shown in Appendix \ref{sec:appendix-jap-exp-setting}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Baselines}
\label{subsec:baselines}

\review[myblue]{MAB-4}{We prepared comparison methods for this word usage classification task. In this experiment, all models address the same problem: classifying a target word in the given sentence as either standard or non-standard.} We evaluate the performance of the model relative to two baseline methods as detailed below.

\begin{enumerate}
\item \wordtovec: This method was proposed by \citeA{aoki2019}, and models the non-standardness of a target word in context according to a weighted average of the dot-product between the word embedding for the target word and word embeddings for the surrounding words (based on a fixed context window size).
This method requires a global threshold to classify whether a given target word is used in a standard way or not. In the experiment, the threshold is determined by 10-fold cross validation over the data.
In comparison, our model does not rely on any such threshold for classification as it is trained directly over the end-task of standard vs.\ non-standard usage prediction.

\item \PreTrained[base]{T5}: To investigate the effectiveness of a pre-trained masked language model for this task, we employed Text-to-Text Transfer Transformer (T5) \cite{T5}, a large language model that does not incorporate a specific masked language modeling architecture within its design.
We used a pre-trained Japanese \PreTrained[base]{T5},\footnote{\url{https://huggingface.co/sonoisa/t5-base-japanese}} which was trained over Japanese Wikipedia and the Japanese portion of CC-100 and OSCAR. 
Note that we did not use T5's original decoder, relying solely on the encoder's hidden layers for this task. For a fair comparison, we added the same LM head networks used in RoBERTa (Section \ref{subsec:context2target}) to the encoder of T5, applied the same classifier layer afterward, and incorporated IN and OUT embeddings in the same manner as in the proposed model. \review[mygreen]{AS-2-2}{As with the proposed model, we freeze the pre-trained weights of the T5 encoder and word embeddings, resulting in the same set of trainable parameters.} The training corpora and parameters were likewise consistent with those used in the proposed model. 
\end{enumerate}

We also report the metrics produced by large language models (LLMs), such as ChatGPT \cite{chatgpt}. However, it is important to note that direct comparison with these models is not feasible due to the financial and computational costs associated with applying such methods to large-scale text streams, like those from Twitter or Reddit. These limitations result in a lack of scalability and make it challenging for these methods to rank examples based on their level of non-standardness.
For the comparison with the LLM based method in our task, we employed GPT-4\footnote{Specifically, we used the \url{gpt-4-0613} model in batch inference.} \cite{gpt4} and Swallow-70b instruction model \cite{swallow}, both of which are multilingual models capable of handling Japanese texts. In these methods, we instructed the model to determine whether the target word in the input text was used in a standard or non-standard way. For constructing a prompt, we incorporated a few-shot approach \cite{prompt-tips,ja-prompt}, providing examples that included both standard and non-standard usages to boost the performance. The concrete prompt used in the experiment is shown in Appendix \ref{sec:appendix-prompt}.
Responses indicating agreement or disagreement were converted into binary labels for evaluations. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Results}

\tabref{result:japanese-results} shows the results on the Japanese Twitter dataset.
The proposed method achieves overall best performance, balancing high accuracy, F1 score, and average precision.
The proposed model outperformed the second-best baseline, \PreTrained[base]{T5}, by a margin of 1.2 to 2.5 points across metrics.
Based on the results, our proposed method outperforms all baseline models in terms of accuracy, F1 score and average precision at a level of statistical significance ($p < .01$) by permutation test.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 1
\begin{table}[b]
\input{06table01.tex}%
\caption{Experimental results on the Japanese Twitter dataset. {\bf Bold} indicates the best score.}
\label{result:japanese-results} 
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Compared to \wordtovec, \PreTrained[base]{T5} and the proposed model achieves better results in major metrics, highlighting the importance of an explicit word usage classification layer in more parameterized pre-trained language models and pseudo-label training.
Specifically, \review[mygreen]{AS-4-7}{when evaluating accuracy in relation to the number of words in a sentence}, the top 20\% \review[mygreen]{AS-4-7}{of longer sentences, containing the most words}, achieved accuracies of .810 and .830 in the \wordtovec method and the proposed method respectively, while the bottom 20\% \review[mygreen]{AS-4-7}{of shorter sentences, containing the fewest words,} had accuracies of .781 and .793, respectively. This indicates that the absolute difference in accuracy is greater in the top 20\%.
One possible explanation for this is that \wordtovec method has a limitation due to its cut-off window, where the capacity of the model is restricted by the size of its context window, resulting in the discarding of valuable information in sentences that contain more words than the window can accommodate. In contrast, our proposed model fully leverages the contextualized representation from the bi-directional transformer, leading to higher accuracy for longer sentences.
\review[mygreen]{AS-4-8}{Shorter sentences were more difficult to classify for both methods due to the lack of contextual information, leading to lower performance. This aligns with the observations made by \citeA{pei-etal-2019-slang} in slang detection.}
Compared to \PreTrained[base]{T5}, the proposed model outperformed the all the evaluation metrics suggesting that an explicit masked language modeling architecture helps in this task.
\review[mypurple]{AP-4-1}{GPT-4 and Swallow did not perform well in the classification.}
Further prompt engineering could help those models achieve better quality, but this is beyond the scope of this paper.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 2
\begin{table}[b]
\input{06table02.tex}%
\hangcaption{Example sentences and corresponding model predictions. \textbf{Bold} indicates the target word. \checkmark \ denotes a correct model prediction.}
\label{result:ja-quality} 
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\review[mybrown]{BP-2-2}{\tabref{result:ja-quality} shows a quality comparison of each method. In contextually rich examples, such as the first and second examples, all models were able to correctly predict the target words. These sentences provide strong contextual cues that make it easier for the models to infer the intended meaning of the target words. \wordtovec model fails to predict correctly when the target word appears at the beginning or end positions in the sentence. This limitation is due to the cut-off window inherent in the Skip-gram architecture, which restricts the contextual information to a fixed window size around the target word. When the target word is near the sentence boundaries, the model lacks sufficient context on one side, leading to incorrect or ambiguous interpretations. 
\PreTrained[base]{T5} model and the proposed method demonstrate superior performance in handling abbreviations and domain-specific terms, particularly in cases involving ``支部'', which refers to the platform name ``pixiv'' in these examples.
All models struggle with short sentences that lack sufficient context or that incorporate commonly used expressions from the internet.
As \citeA{pei-etal-2019-slang} reported, investigating word usages for short sentences is generally difficult. 
In the latter case, we hypothesize that this issue arises from both the training corpora and the pre-trained knowledge of LLMs.
This problem, particularly in the context of evolving word senses, is closely related to diachronic semantic change \cite{www2015-diachronic,hamilton-etal-2016-diachronic,kutuzov-etal-2018-diachronic}, a research field that investigates how the meanings of words shift over time. A potential solution may involve incorporating historical corpora, such as COHA,\footnote{\url{https://www.english-corpora.org/coha}} to better capture these temporal variations in meaning \cite{hamilton-etal-2016-diachronic,sommerauer-fokkens-2019-conceptual,martinc-etal-2020-leveraging,nagata-etal-2023-variance}. However, such language resources are not available for the Japanese language. Alternatively, introducing a bias toward formal language by integrating formal written texts, such as newspaper articles, into the training data could help further mitigate this issue, which we propose as a direction for future research.}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 3
\begin{table}[b]
\input{06table03.tex}%
\caption{Ablation study on Japanese Twitter dataset. {\bf Bold} indicates the best score.}
\label{result:japanese-ablation} 
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\tabref{result:japanese-ablation} shows the ablation study to understand the effectiveness of masked language model and word embeddings.
All the ablation points demonstrated a statistical significance compared to the proposed model, with a level of $p < .01$ according to permutation test.
As detailed in \tabref{result:japanese-ablation}, utilizing both contextualized representations and word embeddings boosts classification performance, suggesting that it is helpful for the model to have features from both \contexttotarget prediction and \targettocontext for this task.
We also found that finetuning the LM head, which directly models a target word prediction (Section \ref{subsec:context2target}), was the second most crucial factor.
This suggests that freezing the LM head restricts its potential ability to understand and distinguish given usages though pseudo-label classification.
While the presence of the LM head itself shows the statistical margin, it was the least impactful compared to other ablation points.
Overall, the ablation study indicates that both the masked language model and word embeddings significantly enhance task performance and these findings suggest that leveraging the full capabilities of the pre-trained prediction component, including the LM head, is important for better performance.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 4
\begin{figure}[t]
\begin{center}
\includegraphics{32-1ia6f4.eps}
\end{center}
\caption{Relationship between training dataset size and F-1 Score.}
\label{fig:ja-dataset}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\review[myorange]{BS-3}{\figref{fig:ja-dataset} shows a performance comparison based on the volume of the training dataset, where the dataset size increases from 100,000 to 1,000,000 in increments of 100,000 measured by F1 scores.
The reported F1 scores represent the average of three runs with different random seeds.
As the dataset size increases, the F1 score generally improves, though there are fluctuations at certain points.
The lowest performance is observed when the dataset size is 200,000, with an F1 score of 0.795.
This slight dip indicates that increasing the data size does not always result in immediate performance gains.
However, as the dataset size continues to increase, the F1 score shows a general upward trend, peaking at a dataset size of 800,000 with an F1 score of 0.828, which represents the highest performance in this experiment.
Beyond 800,000, the F1 score slightly decreases to 0.820 at 900,000 and then stabilizes at 0.821 with a dataset size of 1,000,000.
This suggests that beyond 800,000, adding more data yields only marginal improvements in performance.
Overall, the figure illustrates that while increasing the dataset size generally enhances performance, there is a point of diminishing returns where further data increases yield minimal gains.}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Error Analysis}
\label{subsec:error-ja} 
We analyzed the error cases to investigate 
\pagebreak
what misclassifications were made by the models. For the analysis, we picked the \PreTrained[base]{T5} model and the proposed model that performed the best F1 score on the testing set.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 4
\begin{table}[b]
\input{06table04.tex}%
\vspace{-20pt}
\caption{Confusion matrices for Japanese Twitter experiment.}
\label{results:confusion-ja}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\tabref{results:confusion-ja} shows confusion matrix for each model to analyze classification errors. 
\review[mypurple]{AP-3-2}{In this study, we regard positive as non-standard usages and negative as standard usages.}
The top right cell represents the false-positive cases, where the model incorrectly predicted a standard instance as non-standard, while the bottom left cell represents the false-negative cases, where the model failed to detect a non-standard instance, incorrectly labeling it as standard.
\wordtovec model, shown as (a) in \tabref{results:confusion-ja}, produces a high rate of false positives, indicating frequent misclassification of standard instances as non-standard. \PreTrained[base]{T5} model, shown as (b), achieves the lowest false-positive rate, but the model struggles with detecting non-standard instances, resulting in the highest rate of false negatives.
The proposed model, shown as (c), offers a balanced approach, reducing false negatives more effectively than the other models, which suggests it is particularly strong at identifying non-standard instances. Although the proposed model does \review[mygreen]{AS-4-12}{not} achieve the lowest number of false positives, its overall performance in minimizing misclassifications makes it the most reliable model for distinguishing between standard and non-standard word usages in Japanese Twitter data.
GPT-4, shown as (d), tends not to produce the non-standard label as frequently as other models, yet it achieves the best precision for this \review[mygreen]{AS-4-9}{label}. This indicates that the model only detects non-standard usage when it has high confidence in Japanese texts.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 5
\begin{table}[t]
\input{06table05.tex}%
\hangcaption{Example false-positive and false-negative cases by the proposed method. \review[myorange]{BS-4}{\textbf{Bold}} indicates the target word.}
\label{result:ja-errors} 
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\tabref{result:ja-errors} shows example error cases produced by the proposed model for each failure category. 
The top three word types for misclassification are ``藁'', ``ピザ'', and ``蟹'' for false-positive cases, and ``円盤'', ``沼'', and ``草'' for false-negative cases. These top three word types account for 46.9\% of the total false-positive errors and 28.7\% of the total false-negative errors. \review[mygreen]{AS-4-10}{We observed a false-positive pattern in narrative cases, such as case (b) in \tabref{result:ja-errors}, where the model tends to classify a sentence as non-standard if it contains dialog between characters or humans. However, this pattern did not represent a major cause of the overall false-positive cases.}
For false-negative errors, the model tends to ignore phrasal slang that is often used on the web, such as in case (e) ``沼にはまる'' (becoming enthusiastic about something)'' or case (f) ``草生える'' (indicating laughter in a similar fashion to ``lol'', laughing out loud, in English).
To prevent such failures, we need to carefully choose the corpora for the pre-trained model and pseudo-label training to avoid including examples that could introduce a bias toward net slang in the masked language model and the usage classifier.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Non-standard Usage Discovery Task}

\review[myblue]{MAB-4}{In the previous experiment, we addressed the word usage classification task using a pre-annotated dataset. For practical applications, we extend this task into a non-standard usage discovery task by applying the classification over unannotated corpora to all possible candidate words, selecting sentences and their associated target words with the highest scores for the non-standard label. In this experiment, we utilize Reddit\footnote{\url{https://www.reddit.com}} as a data source to validate our method for the discovery task through crowd-sourced evaluation.}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Data Collection}

This section outlines the detailed data creation process. 
We collect sentences from 
Reddit posts during 2014 for annotation, and run three kinds of models, \wordtovec, \PreTrained[base]{T5} and
our proposed model, over the candidate posts.
After aggregating detected non-standard usages obtained by those methods, we perform human annotation to the results via crowd-sourcing and regard annotated sentences as final dataset.
Reddit is a social media platform and has various communities called 
subreddits which are dedicated to a particular topic.
We \review[mygreen]{AS-4-12}{assume} that each subreddit has its own slang or jargon related to 
the community, which gives us some non-standard usages of certain words.
The processes involved in this procedure will be explained in detail below.

We first select five subreddits for our experiments as shown in \tabref{table:subreddit-statistics}.
Based on \citeA{del-tredici-fernandez-2018-road}, given that ``small-to-medium-sized'' communities are more prone to lexical innovations than larger communities with 15 million users, we selected subreddits that fits this size.
In addition to a niche community, \rhiphopheads, we also 
include subreddits of a more general nature such as \rchan or \rteenagers to 
add some diversity, and \rireland or \raustralia to investigate regional 
variations of English.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 6
\begin{table}[b]
\input{06table06.tex}%
\caption{Selected subreddits and its statistics.}
\label{table:subreddit-statistics} 
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We next create a set of word tokens (and their associated sentences) for 
a subreddit where the words are used as a noun, \review[mypurple]{AP-1-1}{as in the previous experiment, following \citeA{aoki2019}.}  We use the sentence 
chunker from nltk \cite{bird-2006-nltk}, lowercase all words and perform 
part-of-speech tagging with TreeTagger \cite{tree-tagger}.

We filter out nouns that do not occur in COCA,\footnote{\url{https://www.english-corpora.org/coca}} the corpus of contemporary American English, as we focus on unusual usage of an in-vocabulary word.


We run baselines methods, \wordtovec and \PreTrained[base]{T5} and our proposed model over the set of words 
and sentences, and collect non-standard usages of word type
that are predicted by the models. Note that a word type is considered 
non-standard by a model only if it has at least 10 non-standard usages 
by different users.
As a result, we have 40--1,200 candidate word types per subreddits.


Lastly we randomly select five different word types from each subreddits per method, resulting in a total of 15 word types. Using these 15 word types, we extract the 10 most non-standard usages/sentences detected by each method. In total, we select 75 word types, yielding 750 usages per method and resulting in a dataset of 2,250 sentences.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Human Annotation}

We perform human annotation of the collected data using Amazon
Mechanical Turk (Mturk).\footnote{\url{https://www.mturk.com}} In this
annotation, each worker is asked whether the given target word in a
sentence is used in a standard or non-standard way, on a scale of 1 to
5. In more detail, workers are asked to mark non-standard usages as 5 (`Very Unusual')
or 4 (`Unusual') for the given target word, and 2 (`Natural') or 1
(`Very Natural') otherwise. 
The slider is set to a default position of 3, which is not used in the annotation task, and the system automatically blocks submission if the worker does not move the slider towards either 1 or 5.
A snapshot of the annotation screen is shown in \figref{figure:annotation-screen}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% fig. 5
\begin{figure}[t]
\begin{center}
\includegraphics{32-1ia6f5.eps}
\end{center}
\caption{Example MTurk snapshot. Workers are required to move the slider to submit their responses.}
\label{figure:annotation-screen} 
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

In this annotation, workers are asked to annotate 32 sentences
in a HIT (= annotation session).
We include two quality control items in
each set, in the form of one sentence extracted from English Wikipedia
(which workers needed to annotate as 2 or 1) and one sentence from
English Wikipedia where a noun was randomly replaced with the target
word (which workers needed to annotate as 4 or 5). 
We employed several filtering strategies to ensure higher quality from workers: a HIT approval rate greater than 95\%, more than 100 accepted HITs, and United States as location.
We rejected HITs where one of the quality control items was annotated other than expected.\footnote{We rejected HITs that failed both control questions for the first 47\% of the tasks. However, this restriction yielded a 73\% rejection rate, making it difficult to collect sufficient data. Hence, we relaxed the conditions thereafter.}
In total, 273 workers participated in the combined annotation task, and five different workers annotated each sentence.\footnote{Note that we allocate several workers per instance to alleviate individual variation and investigate the majority of the understanding for the term as the annotation task depends on the subjectivity or background knowledge of the workers.}
The average time for workers to complete a HIT was 37.12 minutes, which translates to approximately 1.16 minutes per question.
To mitigate individual variation in the annotated labels, we binarized the labels by converting 5 (`Very Unusual')
or 4 (`Unusual') into a non-standard label and 2 (`Natural') or 1
(`Very Natural') into a standard label as a post-process.
We then assessed inter-annotator agreement using Fleiss' Kappa \cite{fleiss1971} to investigate the quality. The agreement study yielded a score of 0.1129, which falls into the category of slight agreement \cite{inter-annotator}, indicating that the task involves a certain level of subjectivity among annotators. 

For creating the final dataset, we extracted sentences where at least 4 workers agreed on either a standard or non-standard label and discarded the others. It is important to note that this process is a trade-off between the quantity and quality of the dataset; we discarded 940 examples that potentially contains gold labels. However, given the fact that the annotation task depends on the subjectivity of the worker, we decided to prioritize quality over quantity. As a result, 1,076 examples were labeled as standard, and 234 examples were labeled as non-standard. \tabref{results:annotation-details} shows detailed annotation results.
As illustrated in the table, \rchan contains the most non-standard usages in the dataset, whereas \raustralia reflects more common usages.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 7
\begin{table}[t]
\input{06table07.tex}%
\caption{The details of the labels for the created dataset.}
\label{results:annotation-details}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 8
\begin{table}[t]
\input{06table08.tex}%
\hangcaption{Example sentences that are annotated as non-standard by at least 4 workers. {\bf Bold} indicates the target word. Profanities have been redacted. Explanation of each usage: (a) a dominant or assertive male; (b) \textit{Method Man}, a famous rapper in United States; (c) an abbreviation of general as in General Education; and (d) \textit{Celtic Tiger}, the rapid economic growth in Ireland from the 19th--20th century; and (e) an abbreviation of graduated filter. }
\label{result:english-examples} 
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Example non-standard word usages are shown in \tabref{result:english-examples}.
As the examples demonstrate, each usage somewhat reflects the culture of a particular community; \rchan and \rteenagers tend to include more slang, while \rhiphopheads is related to a specific music genre. 
We found that \textit{gen} in case (d) in \tabref{result:english-examples} used in \rteenagers, refers to ``general'', as in General Education, often abbreviated as ``Gen Ed'' in the context of curriculum or courses in a university.
We also found the term \ex{tiger} in case (e) is often used to refer to
\ex{Celtic Tiger} in \rireland that reflects its geographical information about the community.
Contrary to our expectations, however, we did not observe many dialectal usages in \raustralia and \rireland communities.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Model Details}

For the English experiments, we implemented the proposed method using English resources. We trained word embeddings on the COCA and trained the classifier using the pre-trained English \PreTrained[base]{RoBERTa}\footnote{\url{https://huggingface.co/FacebookAI/roberta-base}} on the ukWaC corpus \cite{ukwac}. Similar to the Japanese Twitter experiment, we generated one million standard and pseudo-examples from the corpus, along with 1,000 sentences for validation data. The same hyperparameters as those used in the Japanese experiment were applied, and we selected the model based on its best performance on the validation dataset.
Other training details are shown in Appendix \ref{sec:appendix-en-exp-setting}.

For \wordtovec baseline, we utilized the embeddings trained in the aforementioned process with a threshold of $-0.02$ determined based on the validation dataset described previously. 
\review[mygreen]{AS-4-11}{Unlike the previous experiment on Japanese dataset, where we determined the value based on 10-fold cross-validation over test data, real-world applications like this English experiment require the model to have a single threshold applicable to all sentences.}
We also employed the pre-trained English \PreTrained[base]{T5} model\footnote{\url{https://huggingface.co/google-t5/t5-base}} and followed the same training strategy as the proposed model.\footnote{\review[mygreen]{AS-2-3}{Note that the pre-training corpora for \PreTrained[base]{RoBERTa} and \PreTrained[base]{T5} differ, complicating direct comparisons between the models. Specifically, \PreTrained[base]{RoBERTa} is trained on BookCorpus, Wikipedia, CC-News, OpenWebText, and Stories, while \PreTrained[base]{T5} uses the Colossal Clean Crawled Corpus (C4) and Wikipedia. These differences in training data likely result in distinct linguistic patterns and biases, impacting their performance on downstream tasks.}}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Evaluation Metrics}
Using the newly annotated dataset, we investigate the usage classification performance of the baselines as well as our proposed model.
As in the previous experiment, we used accuracy, precision, recall, F1 score, and average precision to evaluate the rankings, with results from GPT-4 provided as a reference.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 9
\begin{table}[b]
\input{06table09.tex}%
\hangcaption{Experimental results on the English Reddit Dataset. {\bf Bold} indicates the best score. $\dagger$ indicates there is a statistical significance ($p < .01$) against proposed method. }
\label{result:english-results} 
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Results}

\tabref{result:english-results} shows the results on the English Reddit
dataset.  Our proposed model outperforms baselines model on all the major evaluation metrics.
The absolute improvement between the second-best baseline and the proposed model was 9.3, 3.0 and 5.5 points for accuracy, F1 score and average precision respectively.
We observed a statistical significance with a level of $p < .01$ for accuracy and average precision based on permutation test. There is no statistical significance observed for the F1 score.

As we can see from \tabref{result:english-results}, \wordtovec suffers from overall classification performance.
One reason for this is that the \wordtovec method requires a global threshold for decision-making.  In \wordtovec method, it is likely that a single global value cannot effectively work for all examples, and we might need to fine-tune the threshold based on the specific word. In contrast, our proposed pseudo-data based method does not require any threshold tuning for classification achieving the best accuracy and proving to be effective for this task. Compared to \PreTrained[base]{T5}, our proposed method achieves higher evaluation result for across all metrics, highlighting the importance of pre-trained masked language models for distinguishing word usages.
On the contrary to the experiment on the Japanese dataset, GPT-4 achieves the best performance across all the metrics. Although the technical report \cite{gpt4} does not explicitly mention about the training data for GPT-4, the model demonstrates a stronger capability in handling English texts than other languages as reported by its higher performance on language understanding tasks such as GLEU \cite{wang2018glue}. This tendency for higher performance on English also applies to our word usage classification task.
However, as we mentioned in Section \ref{subsec:baselines}, using LLMs is not scalable as the current prompt only produces a binary label for the given input string, making it difficult to determine which specific usage is more likely to be non-standard. In addition, inference with GPT-4 is financially expensive and not feasible for large text streams, such as those on Twitter or Reddit.\footnote{As of August 2024, it would approximately cost over \$50,000 just to apply the classification to the examples that our model processed to create the dataset.} 
As noted in the previous experiment, prompt engineering to boost the performance or teach the model to produce a scalar value is out of the scope of this paper. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 10
\begin{table}[b]
\input{06table10.tex}%
\hangcaption{Example sentences and corresponding model predictions. \textbf{Bold} indicates the target word. \checkmark \ denotes a correct model prediction.}
\label{result:en-quality} 
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\review[mybrown]{BP-2-2}{\tabref{result:en-quality} shows a quality comparison of each method. All models made correct predictions for the first two sentences. Interestingly, these sentences contain metaphorical expressions, such as ``gave me cancer'' and ``mad as hell''. These phrases are not literal but are used to convey strong emotions or negative experiences. Despite the figurative language, all models successfully captured the intended meaning, showing their ability to handle metaphorical usage.
The proposed method and GPT-4 correctly predict certain challenging sentences, even for the typo ``ton'', which was originally meant to be ``ten''.
None of the methods correctly predicted non-standard usages\footnote{Note that there are cases where the model's output is standard, yet the annotation label is non-standard. This occurs because we collect the top 10 usages sorted by the scores for non-standard labels; however, not all usages are classified as non-standard if the score for a usage does not meet the threshold.} when the literal meaning also fits the context. For instance, in \tabref{result:en-quality}, the word ``origin'' refers to a gaming platform, yet its literal meaning fits the sentence contextually. Similarly, the word ``text'', commonly used in digital communication to mean sending a message, was not correctly identified despite its increasing prevalence in internet language.
Developing more robust models to address these cases is left for future work.
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 11
\begin{table}[b]
\input{06table11.tex}%
\caption{Evaluation metrics for each subreddit by the proposed method.}
\label{result:english-results-category} 
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\tabref{result:english-results-category} shows the evaluation metrics for each subreddit by the proposed model.
Overall, the model performs inconsistently across subreddits, with \rchan showing strong precision but weak recall, \rhiphopheads offering a more balanced performance, and \rireland, \rteenagers, and \raustralia demonstrates high accuracy but low precision and recall. 
\review[mygreen]{AS-3-2}{One possible reason for this tendency in accuracy is the imbalance} in the dataset containing non-standard usages; \rchan and \rhiphopheads contain the most non-standard labels, \review[mygreen]{AS-3-2}{with 104 and 40 non-standard usages respectively}, while \raustralia and \rteenagers contain the least, \review[mygreen]{AS-3-2}{with 28 and 30 non-standard usages respectively}, resulting in higher accuracy for subreddits with more standard usages.

\tabref{result:english-hero-cases} illustrates example non-standard usages detected by the proposed model.
As shown in case (a--d) in \tabref{result:english-hero-cases}, the proposed model successfully detects net slangs or non-standard usages across different subreddits. Interestingly, as shown in cases (e--g), the model detects the same word \textit{tier} as non-standard, which may indicate a signal that the term is likely a more general slang term rather than community-specific jargon \cite{del-tredici-fernandez-2018-road}. Additionally, as shown in cases (h--f), the model is able to detect typos labeling them as non-standard suggesting that it could potentially be applied to grammatical error correction at the lexical level \cite{ng-etal-2014-conll,kiyono-etal-2019-empirical,kaneko-etal-2020-encoder} or data cleaning for further NLP applications \cite{saito-etal-2014-morphological,saito-etal-2017-improving,bolding-etal-2023-ask}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 12
\begin{table}[t]
\input{06table12.tex}%
\hangcaption{Example detected non-standard usages. {\bf Bold} indicates the target word. Longer sentences have been shortened and profanities have been redacted. Explanation of each usage: (a) a dominant or assertive male; (b) a toxic person or behaviour; (c) a mobile network company in Ireland; (d) sending a message; (e--g) a slang refers to ranking something or someone by level or quality; (h) a typo for ``guerrilla''; and (f) a typo for ``even''.}
\label{result:english-hero-cases} 
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{Error Analysis}

Similar to the previous experiment with the Twitter dataset, we analyzed the error cases using confusion metrics and conducted an example error case study.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 13
\begin{table}[t]
\input{06table13.tex}%
\vspace{-20pt}
\caption{Confusion matrices for English Reddit experiment.}
\label{results:confusion-en}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\tabref{results:confusion-en} shows confusion matrix for each model to analyze classification errors. 
As noted in Section \ref{subsec:error-ja}, \review[mypurple]{AP-3-2}{we regard positive as non-standard usages and negative as standard usages,} and the top right cell in \tabref{results:confusion-en} represents the false-positive cases (incorrect detection), while the bottom left cell represents the false-negative cases (missed detection).
\wordtovec model, shown as (a) in \tabref{results:confusion-en}, achieves the greatest number of correct detections for the non-standard label, but it also has an increasingly high rate of false positives, which lowers the overall metrics. This may be related to a threshold determined by the validation set, which reveals a discrepancy between the synthetic dataset and actual texts.
\PreTrained[base]{T5} model, shown as (b), produces a similar number of false-negatives compared to the proposed model but has a higher rate of false-positives.
The proposed model, shown as (c), demonstrates a strong performance in distinguishing between standard and non-standard labels. Compared to \wordtovec model and the \PreTrained[base]{T5} model, the proposed model achieves the highest precision for standard label detection, with fewer misclassifications as non-standard. However, similar to the \PreTrained[base]{T5} model, it shows a relatively high number of false negatives for the non-standard label.
As a reference, GPT-4 shown as (d) demonstrates a strong capability in correctly identifying both standard and non-standard labels, with relatively lower rates of false positives and false negatives. This indicates that GPT-4 provides a more accurate and reliable classification performance, making it a robust choice for handling the variability in the dataset depending on the language.
The overall performance suggests that while the proposed model excels at identifying standard labels, it faces difficulties in accurately detecting non-standard labels, similar to the other models compared. The performance of GPT-4 on the English dataset is not as strong as on the Japanese dataset, and these underwhelming outcomes further highlight the complexity of the task, including challenges posed by annotation errors from crowd-sourced evaluations. We will address these issues in the following \review[mygreen]{AS-4-12}{paragraph}, where we conduct an example-wise error analysis that includes a discussion of annotator errors, supported by explanations for each case.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 14
\begin{table}[t]
\input{06table14.tex}%
\hangcaption{Example false-positive and false-negative cases by the proposed method. \textbf{Bold} indicates the target word. Longer sentences have been shortened and profanities have been redacted.}
\label{result:en-errors} 
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\tabref{result:en-errors} shows example error cases produced by the proposed model for each failure category.
As we can see from the results, the false-positive cases include a literal usage of \textit{method} as in case (a) in \tabref{result:en-errors} or \textit{revenue} to mean a financial institution as in case (b). 
However, it also contains annotation errors, such as \textit{mane} in case (c) being used to mean an American rapper ``Gucchi Mane'', or \textit{guise} in case (d) meaning its homophone ``guys''.
For false-negative cases, the model fails to detect non-standard or contextually nuanced usage of certain words. For instance, the model incorrectly treats \textit{hell} in case (e) as a standard usage rather than recognizing it as a name, despite the use of ``his'' as a reference. Likewise, the model fails to identify \textit{alpha} in case (f) as part of a slang expression, highlighting the difficulty the model faces in handling context-specific colloquial usage.
Furthermore, similar to the false positive cases, we have identified annotation errors. Specifically, MTurk workers tend to label usages as non-standard when the target word is used in an interrogative form within a sentence (case (g)) or when the overall meaning of the sentence is unclear or controversial (case (h)).
To prevent such undesired annotation errors in the future, more detailed instructions should be provided on MTurk, explicitly mentioning cases to avoid along with relevant examples.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Work}

\citeA{eisenstein-2013-bad} reported that social media text contains a lot of
non-standard usages, including emoticons and abbreviations, which some
researchers have attempted to normalize at the lexical level \cite{han-baldwin-2011-lexical}.
There has also been work on predicting the meaning of rare or unknown words \cite{stewart-eisenstein-2018-making,Yan_Lin_Hwa_Mert_Ertugrul_Guo_Chung_2020}.
\review[myblue]{MAB-1-2}{One of the related areas of research is slang \cite{kulkarni-wang-2018-simple,pei-etal-2019-slang,sun-etal-2022-semantically}. Particularly, \citeA{sun-etal-2024-toward} focus on broad slang as their research subject, examining not only in-vocabulary words and multi-word expressions such as \ex{sugar daddy}, but also other slang terms like \ex{innit}, used to mean ``isn't it'', or \ex{bruv}, meaning ``brother''.
In our study, we focus solely on the usages of in-vocabulary words appearing in social media texts generated by users, whereas their research mainly deals with subtitles in the movie domain.
Therefore, our research differs from that of \citeA{sun-etal-2024-toward} in terms of the research subject.
Other research focuses on investigating specific language use across communities.} \citeA{del-tredici-fernandez-2018-road} conducted an analysis of language use across different communities, while \citeA{lucy-bamman-2021-characterizing} attempted to utilize BERT to investigate unknown words or meanings in niche communities.

Elsewhere, researchers have focused on in-vocabulary words that have novel or special uses in a particular domain.
\citeA{bamman-etal-2014-distributed} proposed a model to learn word vectors
that reflect geographical differences in language usage.
\citeA{hamilton-etal-2016-inducing} and \citeA{yang-eisenstein-2017-overcoming} reported that, on social
media such as Twitter, words can have opposite sentiment polarities depending on the author or communities that the author belongs.
\review[myblue]{MAB-1-1}{A similar area of research on in-vocabulary words is metaphor detection \cite{choi-etal-2021-melbert,li-etal-2023-metaphor,li-etal-2023-framebert}. Our proposed model also detects metaphoric expressions, such as \ex{hell} in the phrase \ex{getting mad as hell}. However, our research focus is broader than metaphors; it encompasses usages where the meaning cannot be understood without knowledge of the specific context, such as \ex{knob} used to mean ``an obnoxious person''.}
Another related field is diachronic semantic change
\cite{www2015-diachronic,hamilton-etal-2016-diachronic,kutuzov-etal-2018-diachronic,hu-etal-2019-diachronic,giulianelli-etal-2020-analysing,montariol-etal-2021-scalable,inoue-etal-2022-infinite,nagata-etal-2023-variance},
wherein words change in meaning as time passes.



Most related to this work is research on detecting non-standard word usages from a web corpus.
\citeA{lau-etal-2012-word} firstly introduced the task of novel
sense detection, where they attempted to identify novel word senses via word
sense induction through topic modelling, which \citeA{cook-etal-2014-novel} extended by leveraging novelty scores for word senses.
\citeA{aoki-etal-2017-distinguishing,aoki2019} introduced a Japanese Twitter dataset for non-standard word usage detection, and built a model that distinguishes non-standard usages by utilizing the geometry of the \skipgram algorithm.
In this work, we reuse this Twitter dataset for comparative evaluation.
Their approach is based on relative contextual fit for a given target word,
and filtering sentences where the usage of a target word is different from
usages that can be found in a source such as a newspaper.
Although their model can be trained in an unsupervised manner, a threshold for decision boundary is needed for classification, which is a drawback in terms of scalability.
Also, their model relies on a fixed-size context window around the target
word, which prevents the model from accessing valuable information such as clue words around the target word.
Different from this work, our proposed model is an end-to-end method for
non-standard usage detection, that leverages whole context information via a masked language model \cite{devlin-etal-2019-bert,roberta}.
\review[myblue]{MAB-2-1}{Metaphor detection models also incorporate features from masked language models \cite{choi-etal-2021-melbert,li-etal-2023-metaphor,li-etal-2023-framebert}. In these studies, dedicated networks are proposed based on the Metaphor Identification Procedure \cite{mip} and Selectional Preference Violation \cite{spv1,spv2}, utilizing BERT embeddings derived from the output of bi-directional transformer.
In our proposed method, we do not employ networks specialized for these tasks. Our approach differs from these studies in that we introduce features based on \skipgram and utilize information from LM head following the output of the transformer, rather than using the transformer output itself.}
\review[myblue]{MAB-2-3}{Similarly, in the field of diachronic semantic change, many studies utilize contextualized embeddings from models like BERT. However, their focus is on employing word similarity calculations across different time spans \cite{www2015-diachronic,hamilton-etal-2016-diachronic,giulianelli-etal-2020-analysing,martinc-etal-2020-leveraging,nagata-etal-2023-variance} or clustering using topic models \cite{montariol-etal-2021-scalable,inoue-etal-2022-infinite}, which differs from the classification-based approach in our proposed method.
}
\review[myblue]{MAB-2-2}{Slang detection models also incorporate masked language models \cite{zhu-bhat-2021-euphemistic-phrase} or even large language models such as GPT-4 \cite{fillies-paschke-2024-simple,sun-etal-2024-toward}.
These methods rely on slang datasets for training, which are not always available for other languages due to the lack of lexical resources. In contrast, our proposed model uses pseudo-training data, making the approach language-independent and applicable to many languages. This paper demonstrates the approach using English and Japanese as examples, but it is potentially applicable to many other languages, such as Korean \cite{ko_roberta}, Italian,\footnote{\url{https://huggingface.co/osiria/roberta-base-italian}} and Portuguese.\footnote{\url{https://huggingface.co/flax-community/portuguese-roberta-base}}
}
\review[myblue]{MAB-2-3}{A key finding of this paper is the fine-tuning of LM head, rather than the transformer layers of RoBERTa. Although directly accessing the hidden layers of the LM head for classification tasks is uncommon, the LM head itself is employed in language modeling for tasks such as constructive decoding \cite{gera-etal-2023-benefits}, accelerated inference \cite{arora-etal-2022-director,elhoushi-etal-2024-layerskip}, and general masked token prediction \cite{petroni-etal-2019-language,xu-etal-2020-understanding,heinzerling-inui-2021-language,gao-etal-2022-mask}.
}



Our task is different from typical word sense disambiguation \cite{wsd1,ijcai2021p593},
where the system aims to classify a sense of the given target word relative to
a known sense inventory.
The reason for this is that, as \citeA{erk-2006-unknown} mention, it is
intractable to fully enumerate the sense inventory of a word, given the
flexibility of language.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}

In this work, we proposed a RoBERTa-based model for detecting non-standard
usages in social media text.
To deal with the lack of annotated data for non-standard usage detection, we incorporate pseudo-example learning, where we train the model over synthetic examples generated from a corpus.
Experimental results on a Japanese Twitter dataset and English Reddit
dataset show that our proposed method achieves better performance across
a range of evaluation metrics and different languages than baseline methods.
\review[myorange]{BS-2-1}{Our experiments show that fine-tuning LM head and incorporating \skipgram information are essential for improving model performance, enabling better adaptation to non-standard usages. These findings underscore the value of combining different sources of pre-trained knowledge to enhance the model ability to distinguish non-standard usage patterns.
Although we focus on noun cases, the proposed model is able to handle other parts of speech or even phrases, which we leave for future work.
}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.7}
\input{06refs_bbl.tex}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\appendix

\vspace*{-1\Cvs}
\section{Training Details}

\vspace*{-0.25\Cvs}
\subsection{Word Usage Classification}
\label{sec:appendix-jap-exp-setting}

In the proposed method, we apply dropout \cite{dropout} before the ReLU activation in \eqnref{eq:classification} with a dropout rate of 0.1.
The hidden size for the affine transformation in \eqnref{eq:classification} is set to 768.
We used the Adam optimizer \cite{adam}, with learning rate 0.001, learning rate decay factor 0.99, and minibatch size 128.
We trained the model for 30 epochs, and performed model selection based on the model that achieved the best F1 score on the validation dataset.
All experiments were done by using HuggingFace \cite{huggingface}.


For \wordtovec, we followed the best setting of \citeA{aoki-etal-2017-distinguishing} that incorporates not only input embeddings but also output embeddings for the calculation of non-standardness.

For \PreTrained[base]{T5}, we used same configuration to the one used in the proposed model.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\vspace*{-0.5\Cvs}
\subsection{Non-standard Usage Discovery}
\label{sec:appendix-en-exp-setting}

We used the same model configuration to Appendix \ref{sec:appendix-jap-exp-setting}.
Then we select the model that achieves the best F1 score on validation dataset for the proposed model and \PreTrained[base]{T5}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 15
\begin{table}[b]
\input{06table15.tex}%
\caption{Japanese prompt used for GPT-4 and Swallow.}
\label{appendix:ja-prompt} 
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% table 16
\begin{table}[t]
\input{06table16.tex}%
\caption{English prompt used for GPT-4.}
\label{appendix:en-prompt} 
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Prompt for LLMs}
\label{sec:appendix-prompt}

Table \ref{appendix:ja-prompt} and \ref{appendix:en-prompt} show the input prompts used in the Japanese and English experiments, respectively. Note that in the experiments, we replaced \textit{\{sentence\}} and \textit{\{target\_word\}} with the actual input sentence and target word.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Tatsuya Aoki}{%
Tatsuya Aoki received an M.S. in Engineering from Tokyo Institute of Technology. His research interests include natural language processing for semantic changes.
}

\bioauthor[:]{Jey Han Lau}{%
Dr. Lau is a Senior Lecturer in the School of Computing and Information Systems at the University of Melbourne. His research interest is in Natural Language Processing. His research is characterised by a diverse flavour of applications, such as topic models, lexical semantics, text generation and misinformation detection. Some of his work has generated broader community interest beyond academia, e.g., his research in poetry generation and influence operations has been featured in science magazines (New Scientist) and mainstream news media (Guardian and BBC), and his latest collaboration with an artist to develop a personality-mimicking dating chatbot is currently being showcased at NGV Australia.

}

\bioauthor[:]{Hidetaka Kamigaito}{%
Hidetaka Kamigaito received his Ph.D. from Tokyo Institute of Technology. After being an assistant professor at Tokyo Institute of Technology, he is currently an associate professor at Nara Institute of Science and Technology. His current research interests include natural language processing with an emphasis on natural language generation and knowledge graph completion.
}

\bioauthor[:]{Hiroya Takamura}{%
Hiroya Takamura received Master's degree from the University of Tokyo.
He received Ph.D. from Nara Institute of Science and Technology.
He worked as a professor at the Tokyo Institute of Technology,
and is currently a research team leader at AI Research Center of National Institute of Advanced Industrial Science and Technology.
His current research interests include computational linguistics, especially natural language generation.
}
\bioauthor[:]{Tim Baldwin}{%
Tim Baldwin is Provost and Professor of Natural Language Processing at
Mohamed bin Zayed University of Artificial Intelligence (MBZUAI), in
addition to being a Melbourne Laureate Professor in the School of
Computing and Information Systems, The University of Melbourne and
Chief Scientist of LibrAI, a start-up focused on AI safety. He works
on diverse topics in natural language processing and AI.
}

\bioauthor[:]{Manabu Okumura}{%
Manabu Okumura was born in 1962. He earned his B.E., M.E., and Dr. Eng. from Tokyo Institute of Technology in 1984, 1986, and 1989, respectively. He served as an assistant at the Department of Computer Science, Tokyo Institute of Technology, from 1989 to 1992, and as an associate professor at the School of Information Science, Japan Advanced Institute of Science and Technology, from 1992 to 2000. Currently he is a professor at the Institute of Integrated Research at Institute of Science, Tokyo, Japan. His research interests include natural language processing, particularly text summarization, computer-assisted language learning, sentiment analysis, and text data mining.
}
\end{biography}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\biodate



\end{document}





