\documentclass[english]{jnlp_3.8}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{array}
\allowdisplaybreaks

\usepackage{hangcaption_jnlp}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition

%%%%%\usepackage[dvipdfmx]{color}
%%%%%\usepackage{url}
%%%%%\usepackage{otf}
%%%%%\usepackage{amsfonts}
\usepackage{physics}
%%%%%\usepackage{graphics}
%%%%%\usepackage{lscape}
%%%%%\usepackage{color}
%%%%%\usepackage{threeparttable}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{31}
\Number{3}
\Month{September}
\Year{2024}
%%%%\Issuetitle{「　　　　」特集号}

\received{2023}{12}{28}
\revised{2024}{5}{8}
\accepted{2024}{6}{14}



\setcounter{page}{1166}


%%%%   Title, Author, etc.            %%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Weighted Asymmetric Loss for Multi-Label Text Classification on Imbalanced Data}
\eauthor{Yuki Yasuda\affiref{Author_1} \and Taro Miyazaki\affiref{Author_1} \and Jun Goto\affiref{Author_1}} 
\eabstract{
Multi-label text classification, which assigns multiple labels to a single text, is a key task in natural language processing. In this task, a model is often trained on an imbalanced dataset whose label frequencies follow a long-tail distribution. Low-frequency labels that rarely appear in training data have an extremely small number of positive samples, so most of the input samples are negative.
Therefore, the model learns low-frequency labels with the loss value dominated by the negative samples. In this research, we propose a method called weighted asymmetric loss that combines the appearance frequency weight of labels, the weight that suppresses the loss value derived from negative samples, and a label smoothing method in accordance with the co-occurrences of each label. Experimental results demonstrate that the proposed method improves the accuracy compared to existing methods, especially on imbalanced datasets.
}
\ekeywords{Multi-Label Text Classification, Loss Function, Few-shot Learning, Label {\linebreak}Smoothing}

\headauthor{Yasuda et al.}
\headtitle{Weighted Asymmetric Loss for Multi-Label Text Classification on Imbalanced Data}



%%%%     Affiliation     %%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\affilabel{Author_1}{}{NHK Science and Technology Research Laboratories}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S1
\renewcommand{\thefootnote}{\fnsymbol{footnote}}
\footnote[0]{Part of this study was presented at Proceedings of 29th Annual Meeting of the Association for Natural Language Processing \cite{yasuda2023multi}.}
\renewcommand{\thefootnote}{\arabic{footnote}}
\section{Introduction}
Multi-label text classification \cite{zhang2013review,tsoumakas2007multi,NIPS2002_3147da8a} is a key task in natural language processing that is applied to various situations in the real world.
Examples of real-world usage include classification of legal documents \cite{chalkidis-etal-2019-extreme} and automatic diagnosis through medical records \cite{yao2019clinical}.
In multi-label text classification, the task is to assign an appropriate label subset $y^{(i)} = \{y_1^{(i)}, y_2^{(i)}, \cdots , y_N^{(i)}\} \in \{0, 1\}^N$ from predefined labels to document $x^{(i)} \in X$, where $X$ is the set of documents and $N$ is the total number of predefined labels.
The task of multi-label learning is to learn a function $f:\chi^{(i)} \to y^{(i)}$, where $\chi^{(i)}$ $(\chi^{(i)} \subset \mathbb{R}^d)$ denotes the $d$-dimensional feature created from $i$-th document $x^{(i)}$.
Note that a negative sample comprises the pair of a document $x^{(i)}$ and negative labels $y^{(i)}_{n}$ $(y^{(i)}_{n}=0)$.
For the sake of simplicity, we omit $(i)$, which is the index of a sample, in the rest of this paper.

Figure \ref{fig:sample} shows data samples from Reuters-21578 \cite{joachims1998text,10.1145/133160.133172,yang1999re}, which is widely used as a benchmark in multi-label text classification.
Reuters-21578 consists of $21,578$ documents taken from the original Reuters-22173 corpus after the removal of 595 duplicated documents by Lynch and Lewis in 1996.
As shown in the figure, each sample has one or more labels.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F1
\begin{figure}[b]
\begin{center}
\includegraphics{31-3ia12f1.pdf}
\end{center}
  \caption{Samples from Reuters-21578.}
  \label{fig:sample}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



One of the difficulties in text classification in the real world is that datasets often include an imbalanced distribution of labels \cite{he2009learning,wang2020generalizing}.
Datasets, especially those for multi-label text classification, tend to have imbalanced distribution of labels, since the datasets deal with fine-grained labels and a larger number of labels in general.
\citeA{henning-etal-2023-survey} pointed out that long-tailed label distribution is common in multi-label text classification.

As an example, Figure \ref{fig:freq} shows the frequency distribution of labels on Reuters-21578, where we can see that a few labels appear frequently and others appear rarely.
Also, in multi-label text classification, a model learns a mapping function $f$ under the paradigm of binary classification.
Labels that rarely appear in training data, called low-frequency labels, are learned as positive for only a few documents, while labels that frequently appear, called high-frequency labels, are learned as positive for many documents.
Poor variety of input samples with low-frequency labels causes overfitting of models and the accuracy of those labels on test data is diminished.
Thus, it is important to address that imbalanced learning if we want to improve accuracy of a model in multi-label text classification.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F2
\begin{figure}[t]
\begin{center}
\includegraphics{31-3ia12f2.pdf}
\end{center}
  \hangcaption{Label distribution in Reuters-21578. The vertical axis represents the frequency of labels on the dataset, and the horizontal axis represents the index $n$ assigned to the labels.}
  \label{fig:freq}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



In response to this challenge, various loss function methods that weight the loss values so as to suppress the influence of imbalanced distribution have been proposed \cite{cui2019class,wu2020distribution}.
These methods have also been applied to the field of natural language processing, where they have good results \cite{huang-etal-2021-balancing}.
Despite the achievements of these methods, the issues in the tasks remain.
Although multi-label classification often handles datasets with a large label space, typically only a few labels are assigned to one input sample, which results in most of the loss values being derived from negative labels.
In other words, the influence of many negative samples cannot be removed only by adjusting the weight between labels.

In this paper, we propose a loss function that combines weight composed of label frequencies, weight that reduces the influence of negative samples, and weight based on label co-occurrence information.
This loss function aims to suppress the impact of imbalanced datasets including many low-frequency labels on the learning of a model.

Our contributions are as follows:
\begin{itemize}
  \item We propose a loss function called weighted asymmetric loss that considers label frequencies, ground truth values of labels, and co-occurrences of labels to suppress obstruction of learning caused by label imbalance.
  \item We conduct an experiment on widely utilized benchmark datasets and demonstrate that our method outperformed other methods, especially on imbalanced data.
  \item We confirm that methods of adjusting the loss value using weight, including our proposed method, are effective for datasets with an imbalanced label distribution.
\end{itemize}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2
\section{Related Work}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2.1
\subsection{Model Architecture for Multi-label Text Classification}
Many studies have proposed model architectures to improve the quality of multi-label classification.
For example, \citeA{chen2017ensemble} developed a basic encoder-decoder model using a convolutional neural network and recurrent neural network for multi-label classification.
\citeA{adhikari-etal-2019-rethinking} reported that a well-tuned simple Bi-LSTM model can outperform some complex models.

Models considering the relationship between each label have also been proposed.
\citeA{xiao-etal-2019-label} achieved a state-of-the-art performance on the AAPD dataset \cite{yang-etal-2018-sgm} by representing the labels to be classified as word embeddings and having the model learn the relevance to the input samples by means of attention techniques.
\citeA{yang-etal-2018-sgm} proposed a sequence generation model that treats the multi-label text classification task as a sequence generation problem with the aim of considering label relations.

While many methods focusing on architecture have been proposed, such complex approaches tend to require extensive computing resources.
Furthermore, even simple pre-trained models like BERT \cite{devlin-etal-2019-bert} still maintain a state-of-the-art performance on some datasets.
Most existing models for multi-label text classification are trained with binary cross-entropy loss (BCE), which makes them susceptible to label distribution, especially for imbalanced learning.
Therefore, an approach that focuses on the loss function to suppress the effect of imbalanced dataset is required.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2.2
\subsection{Loss Function}
\label{sec:loss}
BCE is widely used in the field of multi-label text classification.
BCE for one sample is defined as
\begin{eqnarray}
  \mathrm{BCE} &=& -\sum_n^N[y_{n}\log{p_n} + (1-y_{n})\log{(1-p_n)}],  \label{eq:bce}
\end{eqnarray}
where $y_{n}$ $(y_{n} \in {\{0,1\}})$ represents the ground-truth value of the $n$-th label.
The $p_n$ $(p_n \in {[0,1]})$ represents the network's output probability that is calculated by a model such as BERT using $\chi$ as input and is corresponding to the $n$-th label.
BCE is the dominant loss function in multi-label text classification, but it is vulnerable to label imbalance effects.

In recent years, a loss function called focal loss (FL) \cite{lin2017focal}, which adjusts weight on the basis of the classification difficulty of input samples, has become mainstream.
FL features coefficients that lower or raise the weight corresponding to samples that are respectively easy or difficult to classify.
FL for one sample is defined as
\begin{eqnarray}
  \mathrm{FL} &=& -\sum_n^N[y_{n}L_{+}^{\mathrm{fl}} + (1-y_{n})L_{-}^{\mathrm{fl}}], \\ \label{eq:fl}
  L_{+}^{\mathrm{fl}} &=& (1-p_n)^{\gamma}\log(p_n), \\
  L_{-}^{\mathrm{fl}} &=& (p_n)^{\gamma}\log(1-p_n),
\end{eqnarray}
where $\gamma$ $(\gamma \geq 0)$ is a hyperparameter that adjusts the strength of the weight.

Class-balanced loss (CBL) \cite{cui2019class} is a technique to weight the loss value by constructing a weight that is inversely proportional to the number of occurrences of each label in a dataset.
CBL for one sample is defined as
\begin{eqnarray}
  \mathrm{CBL} &=&  -\sum_n^N w_n[y_{n} \log{p_n} + (1-y_{n})\log{(1-p_n)}], \label{eq:cbl} \\
  w_n &=& \frac{(1-\beta)}{1-\beta^{c_{n}}}, \label{eq:cb}
\end{eqnarray}
where $c_{n}$ is the number of occurrences of the $n$-th label as positive in training data, and $\beta$ $(\beta \in [0,1))$ is a hyperparameter that adjusts the strength of the weight.
Note that $\beta=0$ corresponds to no weighting and $\beta \to 1$ corresponds to weighting by the inverse of the label frequency.
The weight $w_n$ $(w_n \in (0,1]^N)$ becomes lower when the $n$-th label occurs frequently and is multiplied by the loss value corresponding to the label.

CBL reduces the loss values derived from labels that occur frequently in the dataset.
Due to intrinsic similarities among real-world text data, as the number of samples grows, a sample newly added is a near-duplicate of existing samples.
For example, the specific words related to the ground-truth labels tend to occur in the input sample.
Such redundant information causes diminishing marginal benefits of data \cite{cui2019class}.
CBL can estimate the effective number of samples and reduce redundant information of high-frequency labels.

However, it cannot consider the balance between loss values from positive and negative labels on the same input samples.
The Loss value for one input sample is represented as the summation of loss for all labels.
Since only a few labels are assigned as positive to one input sample in the task, the summation of loss is overwhelmed by the loss derived from negative labels.

Distribution-balanced loss (DBL) \cite{wu2020distribution} was proposed to solve this problem.
DBL is constructed by integrating a weight to suppress the effect of label frequencies and then applying negative-tolerant regularization (NTR).
The weight is calculated by taking the ratio of the reciprocal of the number of label appearances in the batch to the reciprocal of the number of label appearances in the entire data.
Then, NTR suppresses the learning of negative samples by subtracting the value calculated by the output of the model and the hyperparameters.
The loss value can be sharply reduced when the output of the model falls below a certain value.
However, in DBL, there are many hyperparameters that must be predefined in the loss function, which makes it complicated to adjust.
In addition, it is not entirely clear what the values constructed from the plethora of NTR hyperparameters actually mean.

Another method to mitigate the effect of negative samples during training is asymmetric loss (ASL) \cite{Ridnik_2021_ICCV}, which applies asymmetrically shaped weight to the loss values.
ASL for one sample is built on FL and is defined as
\begin{eqnarray}
  \mathrm{ASL} &=& -\sum_n^N[y_{n}L_{+}^{\mathrm{asl}} + (1-y_{n})L_{-}^{\mathrm{asl}}], \\ \label{eq:asl}
  L_{+}^{\mathrm{asl}} &=& (1-p_n)^{\gamma_+}\log(p_n), \\ \label{eq:asf_p}
  L_{-}^{\mathrm{asl}} &=& (p^{(n)}_{\mathrm{m}})^{\gamma_-}\log(1-p^{(n)}_{\mathrm{m}}), \\ \label{eq:asf_n}
  p_{\mathrm{m}}^{(n)} &=& \mathrm{max}(p_n-\mathrm{m},0), \label{eq:ps}
\end{eqnarray}
where, $\gamma_+$ and $\gamma_{-}$ $(0 \leq \gamma_+ < \gamma_{-})$ are hyperparameters that adjust the strength of the weights.
$\mathrm{m}$ in Eq. \ref{eq:ps} is a fixed number that is sufficiently small.
Eq. \ref{eq:asl} is the same as Eq. \ref{eq:fl} in term of FL, but it differs in the separation of $\gamma$ and in using $p_{\mathrm{m}}$ instead of $p$.
The method that separates the attenuation rate $\gamma$ in FL into $\gamma_{+}$ for positive samples and $\gamma_{-}$ for negative samples is called an asymmetric focusing (AF),
and the method using $p^{(n)}_{\mathrm{m}}$ instead of $p_n$ when calculating $L_{-}^{\mathrm{asl}}$ , which is called asymmetric probability shifting (APS), completely discards the loss values from negative samples for which learning has sufficiently progressed.
This selective weighting of the loss value is performed according to whether the label is positive or negative, which makes the learning more meaningful.
However, while ASL is highly effective in suppressing the effect of negative samples, it cannot suppress the effect of label frequency.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2.3
\subsection{Label Dependency}
From a different perspective, methods that utilize dependencies between labels can also help improve the accuracy on multi-label text classification tasks.
\citeA{dembczynski2012label}  pointed out the importance to consider label correlation in multi-label text classification domain.
\citeA{pal2020multi} proposed the model that treats the relationships between each label by means of graph attention networks \cite{veli2018graph}.
\citeA{zhao2024tlc} developed the model that makes clusters of labels and extract the correlation between clusters.
\citeA{song2023label} achieved a state-of-the-art performance on some datasets by the method combining cloze task and multi-label text classification.
On the other hand, many methods that takes into account the relationship between labels have been proposed in the image classification domain \cite{chen2021learning,wang2016cnn,ye2020attention}.

Many methods based on label correlation utilize label embedding technique \cite{zhang-etal-2018-multi} and graph convolutional networks \cite{schlichtkrull2018modeling}.
The method based on arranging ground-truth values directly, for example label smoothing (LS) \cite{Szegedy-labelsmoothing}, is not mentioned so much.
LS can suppress overfitting of a model and calibrate the model \cite{muller2019does}.
We believe that LS can be helpful for improving the accuracy of the model for multi-label text classification.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3
\section{Proposed Method}
We propose a method called weighted asymmetric loss (WASL), which is inspired by the ASL described in Section \ref{sec:loss}.
In our proposed method, $\gamma_{-}$, which decides how strongly to suppress the loss values derived from negative samples in ASL, is weighted by means of class-balanced weight.
We aim to appropriately select loss values from negative labels to suppress and adjust the suppression in accordance with the label frequencies.
Furthermore, by performing label smoothing based on the co-occurrence expressions of the labels, we compensate for the small number of samples of low-frequency labels and suppress overfitting of a model.
We defined WASL as
\begin{eqnarray}
  \mathrm{WASL} &=& - \sum_n^N w^{\prime}_n [y^{\prime}_{n} L_{+} + (1-y^{\prime}_{n})L_{-}], \label{eq:wasl-f} \\
  L_{+} &=& (1-p_n)^{\gamma_{+}}\log(p_n), \label{eq:waslp} \\
  L_{-} &=& (p^{(n)}_{\mathrm{m}})^{\gamma_{-}^{\prime(n)}}\log(1-p^{(n)}_{\mathrm{m}}), \label{eq:wasln}\\
  \gamma_{-}^{\prime(n)} &=& w^{\prime}_n\gamma_{-}, \label{eq:waslg} \\
  w^{\prime}_n &=& \frac{w_{n}N}{\sum_{n}^{N}{w_{n}}}, \label{eq:waslcb}
\end{eqnarray}
where $\gamma_{+}$ and $\gamma_{-}$ $(0 \leq \gamma_{+} < \gamma_{-})$ are the same hyperparameters as used in ASL.
$L_{+}$ is the loss value derived from positive labels and $L_{-}$ is loss value derived from negative labels.
$w_n$ is a class-balancing (CB) weight defined in Eq. \ref{eq:cb}, and $y^{\prime}_n$ represents the smoothed ground truth value.
This method consists of three modules: one for asymmetric focusing, one for class balancing and label smoothing.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.1
\subsection{Asymmetric Focusing Module}
First, we explain the asymmetric focusing (AF) module.
This module calculates loss values $L_{+}$ and $L_{-}$ from positive and negative samples.
The main difference between our proposed method and ASL is that there are class-balanced weight terms $w^{\prime}_n$ in Eqs. \ref{eq:wasl-f} and \ref{eq:waslg}.
Note that $p_\mathrm{m}$ is utilized to discard the influence of negative samples when the output probability of the model is very low, as mentioned in Section \ref{sec:loss}.

Similar to ASL, this module computes weights $(1-p_n)^{\gamma_{+}}$ and $(p^{(n)}_{\mathrm{m}})^{\gamma_{-}^{\prime(n)}}$ that adjust loss values from positive and negative labels, respectively.
The hyperparameter $\gamma_{-}$ that decides the strength of suppression is multiplied by $w^{\prime}_n$.
$w^{\prime}_n$ is re-sampled CB weight, which is low if the label $y_n$ appears frequently in the dataset.
$(p^{(n)}_{\mathrm{m}})^{\gamma_{-}^{\prime(n)}}$ with low $\gamma_{-}^{\prime(n)}$ increases $L_{-}$, while $(p^{(n)}_{\mathrm{m}})^{\gamma_{-}^{\prime(n)}}$ with high $\gamma_{-}^{\prime(n)}$ decreases $L_{-}$.
In other words, $\gamma_{-}^{\prime(n)}$ decides how much suppress the losses derived from negative labels according to the label frequencies.
In the AF used in ASL, the loss value from a negative label is defined as $L_{-}^{\mathrm{asl}} = (p^{(n)}_{\mathrm{m}})^{\gamma_-}\log(1-p^{(n)}_{\mathrm{m}})$, and a constant $\gamma_{-}$ value is utilized to determine the strength of suppression, regardless of label frequency.
As Figure \ref{fig:gamma} shows, with the AF module, loss $L_{-}$ for a negative label is suppressed as the value of $\gamma_{-}^{\prime(n)}$ increases.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F3
\begin{figure}[b]
\begin{center}
\includegraphics{31-3ia12f3.pdf}
\end{center}
  \caption{Relationships between loss for a negative label $L_{-}$, probability $p^{(n)}_{\mathrm{m}}$, and $\gamma_{-}^{\prime(n)}$ in AF module.}
  \label{fig:gamma}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.2
\subsection{Class Balancing Module}
Next, we describe the class-balancing (CB) module.
This module calculates the weight $w^{\prime}_n$ in Eqs. \ref{eq:wasl-f} and \ref{eq:waslg}.
The weight $w^{\prime}_n$ is constructed using normal CB weight $w_n$ based on the number of appearances of labels advocated in CBL.
$w^{\prime}_n$ can apply the weight to the loss from the $n$-th label, like CBL, but also has a role to adjust $\gamma_{-}^{\prime(n)}$ by each label.

Eq. \ref{eq:waslcb} represents the weight redistributed according to the proportion of CB weight of each label.
In the normal CB weight shown in Eq. \ref{eq:cb}, weight differences of each high-frequency label are so low that they disappear in the computing processes.
We resampled the weight as shown in Eq.\ref{eq:waslcb} so that the model can more appropriately consider the differences in label frequencies even for high-frequency labels.
As a label $y_n$ appears frequently in a dataset, $w^{\prime}_n$ becomes lower.
Also, the normal CB weight take values of $1$ or less, but the re-sampled weight $w^{\prime}_n$ can take a value of $1$ or more if the label appearance frequency is sufficiently low.
This allows a wider range of changing for $\gamma_{-}^{\prime(n)}$, which affects the ultimate value of the loss.
$w^{\prime}_n$ can amplify the loss value from positive low-frequency labels while simultaneously suppressing the loss value from negative low-frequency labels.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.3
\subsection{Label Smoothing Module}
Finally, we discuss the label smoothing (LS) module.
Unfortunately, weight adjustment by class balancing and asymmetric focusing does not fully solve the problem of the small number of input samples corresponding to low-frequency labels.
Therefore, we perform LS \cite{Szegedy-labelsmoothing} with the label co-occurrence information set as prior probabilities to suppress overfitting of the model.
Label smoothing here is defined as
\begin{eqnarray}
  y^{\prime}_{n} &=& (1-\alpha)y_n + \alpha \, \mathrm{norm}(o_n), \label{eq:ls} \\
  \mathrm{norm}(o_n) &=& \frac{o_n - \mathrm{min}(\vb*{o})}{\mathrm{max}(\vb*{o}) - \mathrm{min}(\vb*{o})}, \\
  \vb*{o} &=& \vb*{y} \cdot \vb*{P},
\end{eqnarray}
where $\alpha$ $(\alpha \in [0,1])$ is a hyperparameter that determines the degree of smoothing and $\mathrm{norm}(\cdot)$ is the min-max normalization function.
Also, $\vb*{y}$ $(\vb*{y} \in {\{0,1\}}^N)$ is a multi-hot vector for the input sample and $\vb*{P}$ indicates a square matrix of positive pointwise mutual information (PPMI) \cite{church-hanks-1990-word,niwa-nitta-1994-co} calculated from the number of co-occurrences of each label.

In normal LS \cite{Szegedy-labelsmoothing}, uniform distribution $\frac{1}{N}$ is utilized instead of $\mathrm{norm}(o_n)$ in Eq. \ref{eq:ls} under the assumption that the labels are unrelated to each other.
However, it is unlikely that the distribution of negative labels is uniform, as labels that express similar concepts are highly related and likely assigned to the same input sample.
Thus, we regard semantic connections as co-occurrences of labels and decide to approximately represent the distribution of negative labels as PPMI.
The PPMI score for each label is given by
\begin{eqnarray}
  P_{l_i,l_j} = \mathrm{max}(0,\log_{2}\frac{C(l_i,l_j)}{C(l_i)C(l_j)}), \label{eq:ppmi}
\end{eqnarray}
where $P_{l_i,l_j}$ is the element of $\vb*{P}$ corresponding to $l_i$ and $l_j$ $(l_i,l_j \in L)$ that refer to the $i$-th and $j$-th labels.
$C(l_i)$ represents the number of samples that is given the $i$-th label and $C(l_j)$ represents the number of samples that is given the $j$-th label respectively.
In addition, $C(l_i,l_j)$ represents the number of samples that is given both the $i$-th label and the $j$-th label.
Note that we compute the PPMI matrix from training data only.

Since $y_{n}$ can only take the values $1$ or $0$, only $L_{+}$ or $L_{-}$ is added to the ultimate value of loss.
Such loss calculation ignores the correlation between labels, which is the feature of the multi-label classification task.
In our method, negative labels co-occurring with ground-truth labels in other samples are regarded as related labels, and relation score $\alpha \mathrm{norm}(o_n)$ is distributed only to the ground-truth values of the related labels (Fig. \ref{fig:targets}), where ground-truth values represent $y^{\prime}_{n}$ $(y^{\prime}_{n} \in {[0,1]})$ for a model to fit.
In other words, a small amount of loss values as positive label about contextually relevant labels is added to the ultimate loss value, even if they are not truly positive samples.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F4
\begin{figure}[b]
\begin{center}
\includegraphics{31-3ia12f4.pdf}
\end{center}
  \caption{Hard target and smoothed ground truth values $y^{\prime}_{n}$.}
  \label{fig:targets}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



As shown in Figure \ref{fig:targets}, for example, the labels ``wheat'', ``flour'', and ``grain'' tend to co-occur with each other in the training data.
If ``wheat'' and ``grain'' are positive labels for an input sample, it is likely to also be associated with the label ``flour''.
Since our method predefines the strength of co-occurrence between labels using the PPMI matrix, some values are distributed to related labels called ``flour'' according to the strength of the co-occurrence.
At this time, no value is distributed to the label ``gold'' because it does not co-occur with ``wheat'' and ``grain'' in the training data.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.4
\subsection{Justification of the Loss Formulation Design}
The justification of the design of our method is supported by the gradient analysis in ASL \cite{Ridnik_2021_ICCV}.
The gradient value for $L_{-}$ is defined as
\begin{eqnarray}
  \frac{dL_{-}}{dz_n} = (p^{(n)}_{\mathrm{m}})^{\gamma_{-}^{\prime(n)}} \lbrack \frac{1}{1-p^{(n)}_{\mathrm{m}}} - \frac{\gamma_{-}^{\prime(n)} \log(1-p^{(n)}_{\mathrm{m}})}{p^{(n)}_{\mathrm{m}}} \rbrack p_n(1-p_n), \label{eq:grad}
\end{eqnarray}
where $z_n$ is the logit, $p_n$ is the output of the model with sigmoid function $\frac{1}{1+e^{-z_n}}$, and $p_{m}$ is defined in Eq. \ref{eq:ps}.
Note that we set $\mathrm{m} = 5 \times 10^{-2}$ obtained by grid search.
Figure \ref{fig:grads} shows the relationship between loss gradient and probability for a negative label.
We can see here that the gradient in our method increases non-linearly, while the gradient in BCE $(\mathrm{m}, \gamma_{-}^{\prime(n)} = 0)$ increases linearly.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F5
\begin{figure}[b]
\begin{center}
\includegraphics{31-3ia12f5.pdf}
\end{center}
  \caption{Loss gradient $\frac{dL_{-}}{dz_n}$ and output probability $p_n$.}
  \label{fig:grads}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


First, when the $p_n$ is very low $(p_n \le \mathrm{m})$, label $y^{\prime}_{n}$ is easy to classify and the loss gradient is ignored in order to focus on the harder sample.
Second, the loss gradient of the negative label with $p_n$ $(\mathrm{m} < p_n \le p_{n}^{*})$, where $p_{n}^{*}$ is defined as the point where $\frac{d}{dp_n}(\frac{dL_{-}}{dz_n})=0$, decreases slowly from $0$ to around $0.7$.
This indicates that as a model fits to a negative label, the gradient of the label is strongly suppressed.
Finally, the gradient decreases sharply if $p_n$ is very high $(p_{n}^{*} < p_n)$.
When a model outputs a large probability for a negative label, it is possible that the label was mislabeled that it should be positive for the sample.
In this case, the gradient for a mislabeled sample should be decreased, since even a small mislabeling rate of negative labels significantly impacts the training in imbalanced datasets.

The value of $\gamma_{-}^{\prime(n)}$ affects the sharpness of the peak of the gradient in our method.
If $\gamma_{-}^{\prime(n)}$ is small, the peak will be soft, and if it is large, it will be sharp.
When the negative label $y^{\prime}_{n}$ is frequently labeled as negative in the dataset, $\gamma_{-}^{\prime(n)}$ becomes high and reduces the gradient for learning a negative label.
This supports our intention of reducing the effect of learning for one sample is dominated by many negative labels by adjusting $\gamma_{-}^{\prime(n)}$ with the labels.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4
\section{Experiment}
We conducted a comparative experiment with baseline methods to investigate the effectiveness of the proposed method.
We utilized models along with other loss function methods utilizing the same classification model on each dataset.
The performance of each method was evaluated five times with different random seeds.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.1
\subsection{Datasets}
Reuters-21578 \cite{joachims1998text,10.1145/133160.133172,yang1999re}, AAPD \cite{yang-etal-2018-sgm}, and EURLEX57K \cite{chalkidis-etal-2019-extreme} were utilized as benchmarks, all of which have a long-tail distribution of labels.
Statistics for each dataset are provided in Table \ref{tab:data}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T1
\begin{table*}[b]
\input{12table01.tex}
  \caption{Dataset statistics.}
  \label{tab:data}
\end{table*}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


Reuters-21578 consists of economics news articles that appeared on the Reuters newswire in 1987 and is characterized by a highly imbalanced distribution of labels.
We applied modApt{\'e} split \cite{apte1994automated} to split the training data and test data in this dataset.
Also, since the validation data is not predefined in Reuters-21578, we constructed it by randomly dividing the training data at a ratio of $9:1$.
AAPD consists of paper abstracts published in the arXiv scientific paper repository, and has a medium imbalance distribution.
EURLEX57K is a huge multi-label text classification dataset consisting of $57,000$ English EU legislative documents and $4,300$ labels from the EUR-LEX portal.
This dataset has many low-frequency labels and is characterized by a highly imbalanced distribution of labels.
It contains not only regular labels but also a large number of zero-shot labels, and we calculated evaluation metrics with all labels that include those labels.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.2
\subsection{Baselines}
As baselines, we selected methods that have achieved state-of-the-art scores for micro-f1, which is a widely recognized metric in multi-label text classification.
We used RoBERTa with CBL \cite{liu2019roberta,huang-etal-2021-balancing} for Reuters-21578, LSAN \cite{xiao-etal-2019-label} for AAPD, and BERT with BCE \cite{devlin-etal-2019-bert,chalkidis-etal-2019-extreme} for EURLEX57K as baseline methods.
In addition, BCE, FL, CBL, CBL-NTR, DBL, and ASL with RoBERTa were used as baseline methods focusing on loss function.
Note that LSAN requires pretrained word vectors, but the developers did not publish the word vectors other than for AAPD.
We utilized word2vec-google-news-300\footnote{\url{https://code.google.com/archive/pw/ord2vec/}} as the pretrained word vectors on Reuters-21578 and EURLEX57K.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.3
\subsection{Experimental Settings}
A neural network was utilized as the classification model in this experiment.
We used RoBERTa \cite{liu2019roberta}, which comprises an encoder part that converts input samples into sentence features and a two-layer feed forward neural network (FFNN) that outputs the score of each label from the sentence features.
The model was implemented using PyTorch \cite{paszke2019pytorch} and Transformers \cite{wolf-etal-2020-transformers}, and the optimization method was AdamW \cite{loshchilov2019decoupled}.

The common hyperparameters in all datasets and all methods are as follows.
Learning epoch was set to $50$, dropout rate was $0.1$, and output threshold was unified to $0.5$.
The model in our experiment outputs a positive label if the probability of the label is greater than a preset threshold (output threshold).
In the main experiment, output threshold was unified to $0.5$, which is the most common setting in multi-label text classification, since the threshold value is not directly related to the performance of a loss function.

In this paper, we used many specific hyperparameters for our method and the baselines, which are described in Appendix \ref{sec:appendix-a} for the sake of simplicity.
All hyperparameters (except the common ones) were determined by using grid search according to micro-f1 on validation data.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.4
\subsection{Metrics}
We used macro-f1 and micro-f1 as evaluation metrics.
Additionally, statistical significance was tested using a permutation test, which is a non-parametric test.
We performed the permutation test under the setting of a one-tailed test with a significance level of $5\%$.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.5
\subsection{Experimental Results}
Table \ref{tab:res} lists the results of this experiment.
The standard deviations of the main experimental results for each model are presented in Table \ref{tab:sd-res}.
As we can see in Table \ref{tab:res}, the proposed method significantly outperformed the baselines on Reuters-21578 and EURLEX57K.
No statistically significant difference was observed between the proposed method and ASL on AAPD.
These results suggest that the proposed method is effective for learning on a imbalanced dataset.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T2
\begin{table*}[b]
\input{12table02.tex}
  \hangcaption{Experimental results. 
  Means of five trials are shown for the proposed method and baselines.
  Metrics with $^*$ indicate that the difference of the metric between the proposed method and the baseline is statistically significant $(p<0.05$).}
  \label{tab:res}
\end{table*}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T3
\begin{table*}[b]
\input{12table03.tex}
  \caption{Standard deviations of main experiment for each method.}
  \label{tab:sd-res}
\end{table*}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


Table \ref{tab:res-groups} shows the results by groups of label frequency for Reuters-21578, AAPD, and EURLEX57K.
We split the labels of each dataset into three groups---high, middle, and low---according to the order of label frequencies.
The high-group consists of labels with frequencies in the top third, while the low-group consists of labels with frequencies in the bottom third.
For example, each group (high, middle, and low) in Reuters-21578 consists of $30$ labels.
As we can see, the proposed method improved the scores of the total label groups compared to the baseline methods, and it especially improved the score of low-frequency labels on Reuters-21578.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T4
\begin{table*}[t]
\input{12table04.tex}
  \caption{Experimental results by groups of label frequency.}
  \label{tab:res-groups}
\end{table*}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5
\section{Discussion}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.1
\subsection{Ablation Study}
Since our proposed method adjusts the loss value using multiple weights, it is important to verify which modules contribute to improving the score.
Therefore, we performed an ablation test, where all hyperparameters in this experiment were set to the same values as in the main experiment.
Table \ref{tab:res-ab} lists the results of our proposed method with one of the modules removed on Reuters-21578.
As we can see, ablating each feature decreases both the macro-f1 and micro-f1 scores, thus demonstrating that all features contribute to improving the classification performance.
While the class balancing module has the lowest impact and contributes little to the scores, the effect of the LS module is large, indicating that it contributes the most to improving the classification accuracy.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T5
\begin{table}[b]
\input{12table05.tex}
    \hangcaption{Experimental results of ablation test on Reuters-21578. 
    Means of three trials are shown for each method.}
    \label{tab:res-ab}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



We next investigated whether the accuracy could be improved by combining the LS module with other loss functions.
All hyperparameters in this experiment were set to the same values as in the main experiment.
As shown in Table \ref{tab:res-ls}, the score improved when we used the existing loss function methods together with the LS module.
However, CBL with LS showed a decrease in score compared to the normal CBL for both macro-f1 and micro-f1.
This suggests that the combination of weights, which is the core concept of the proposed method, may contribute to the increase in score.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T6
\begin{table}[t]
\input{12table06.tex}
 \hangcaption{Experimental results of each method with LS module on Reuters-21578. Means of three trials are shown for the each method.}
    \label{tab:res-ls}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


A simple CB weight $w_n$ increases the loss from low-frequency labels regardless of whether each label is positive or negative.
Also, the LS module creates distributed ground truth value $y^{\prime}_{n}$ $(y^{\prime}_{n} \in [0,1])$ from $y_{n}$ $(y_{n} \in \{0,1\})$.
As a result, the total loss of CBL with LS is dominated by the losses $(y^{\prime}_{n} \log{p_n})$ and $((1-y^{\prime}_{n})\log{(1-p_n)})$ for the low-frequency labels.
Our method has a mechanism that selectively increases or decreases the weight based on whether the label is low-frequency or not, and whether it is positive or negative.
We believe that this selective mechanism contributes to improve performance of the model.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.2
\subsection{Robustness against Hyperparameters}
A common problem with approaches based on loss function is the large number of hyperparameters to tune.
Therefore, it is important to investigate the impact of this on our method as well.
Figures \ref{fig:hyper-ab} and \ref{fig:hyper-g} respectively show the relationship between micro-f1 scores and the transition of hyperparameters on Reuters-21578.
As we can see, the proposed method is relatively robust at low values for hyperparameters $\alpha$ and $\gamma_{-}$, but the classification accuracy of the model drops sharply after a certain value.
Regarding $\alpha$, this is presumably because the model was unable to discriminate truly positive labels due to the increased smoothing strength.
Also, loss values from negative labels are suppressed to such an extent that the model could not learn sufficiently when $\gamma_{-}$ became high.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F6
\begin{figure}[t]
\begin{center}
\includegraphics{31-3ia12f6.pdf}
\end{center}
  \caption{Relationships between micro-f1 scores, $\alpha$, and $\beta$.}
  \label{fig:hyper-ab}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F7
\begin{figure}[t]
\begin{center}
\includegraphics{31-3ia12f7.pdf}
\end{center}
  \caption{Relationship between micro-f1 scores and $\gamma_{-}$.
  Note that $\gamma_{+}$ was fixed at $1$.}
  \label{fig:hyper-g}
{\vskip 10pt}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.3
\subsection{Effects of Threshold}
In the main experiment, the threshold was unified to $0.5$, which is the most common setting, to eliminate the difference in each method.
However, apart from loss function design, it is important to search for thresholds using a validation set to improve the score.
Additional experimental results applying the threshold search method are shown in Table \ref{tab:res-th}.
Each method was evaluated after searching for the best threshold based on micro-f1 using the validation set.
Note that all hyperparameters (except thresholds) in this experiment were set to the same values as in the main experiment.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T7
\begin{table*}[t]
\input{12table07.tex}
  \hangcaption{Experimental results with best thresholds. 
  Means of three trials are shown for the proposed method and the baselines.}
  \label{tab:res-th}
\end{table*}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



Table \ref{tab:res-th} shows that the simplest BCE method provides a strong baseline.
However, we can see that our proposed method outperformed all other methods, except on the EURLEX57K dataset.
In addition, since the best threshold of ASL is higher than the threshold of WASL, we can see that the scheme of ASL increases the sensitivity of the model and causes over confidence.
We believe that combining the weight scheme with suppression from negative labels contributes to increasing scores and suppressing over confidence in the ASL method.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.4
\subsection{Effects of Label Distribution}
\label{sec:20news}
The main experiment was conducted to verify that the proposed method is effective for datasets with an imbalanced label distribution.
Considering application to the real world, a method of intentionally eliminating imbalance in label distribution when constructing training data is also conceivable.
Therefore, in order to clarify the limitations of our method, we conducted an additional experiment using 20 Newsgroups \cite{Lang95}, a dataset with a less imbalanced label distribution. 
The common hyperparameters in this experiment for all methods are as follows.
Learning epoch was set to $50$, dropout rate was $0.1$, and output threshold was unified to $0.5$.
Hyperparameters that are specific for each method were determined by using grid search according to micro-f1 on validation data.
For the sake of simplicity, hyperparameters for each method are described in Appendix \ref{sec:appendix-a}.

The 20 Newsgroups dataset consists of $18,846$ news articles and $20$ groups to classify.
Each group is represented as combination of multiple different topics.
Although the dataset was originally introduced for single-label classification of documents to 20 groups, we used it as a multi-label classification dataset where each topic is a label.
The dataset includes 33 topics as labels and exhibits a well-balanced distribution of labels.
Note that 20 Newsgroups includes only training data and testing data, so we constructed the validation data by randomly dividing the training data at a ratio of $9:1$.

Table \ref{tab:res-20} lists the experimental results for each method on 20 Newsgroups.
The standard deviations of these results for each model are presented in Table \ref{tab:sd-20}.
We performed the permutation test in the same manner as the main experiment.
However, no statistically significant differences were observed between the proposed method and the baselines, except LSAN.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T8
\begin{table}[b]
\input{12table08.tex}
    \hangcaption{Experimental results on 20 Newsgroups. 
    Means of five trials are shown for the proposed method and the baselines.
    Metrics with $^*$ indicate that the difference of the metric between the proposed method and the baseline is statistically significant $(p<0.05$).}
    \label{tab:res-20}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T9
\begin{table}[b]
\input{12table09.tex}
    \caption{Standard deviations of additional experiment for each method on 20 Newsgroups.}
    \label{tab:sd-20}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



As Table \ref{tab:res-20} shows, the proposed method is not effective for datasets with little imbalance in the label distribution.
In contrast, the method combining BERT with BCE, which is one of the simplest methods we examined, has the highest accuracy and outperformed other methods that adjust the loss value using some weight.
However, no statistically significant differences were observed between the method combining BERT with BCE and the other methods using weights.
This suggests the possibility that loss function methods based on weight adjustment, including our proposed method, are effective only for data with an imbalanced label distribution.
However, the proposed method does not have much difference in performance compared to other baselines using RoBERTa, and it is not completely weak for a dataset with a stable label distribution.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.5
\subsection{Limitations}
The methods set as baselines in our experiments have made great achievements in multi-label text classification, but they also include a slightly old method.
This is because these models were selected on the basis of the evaluation metric and on whether the model is widely recognized.
Comparisons with complex large language models have not yet been conducted, and additional research is needed to address this.
Also, as mentioned in Section \ref{sec:20news}, the proposed method is not effective for datasets with a stable label distribution.
Although the degree of weighting can be adjusted by changing the hyperparameters, all weighted loss functions performed best with hyperparameters similar to the other datasets.
The weighting itself may have an adverse effect on a model in the form of noise during learning, but it is still unclear why weighted loss functions cannot outperform BCE, which is the simplest loss function, on the 20 Newsgroups dataset.
Further investigation is needed to determine whether weighting a method can prevent a model from learning on a dataset with stable label distributions.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6
\section{Conclusion}
In this paper, we proposed a loss function called weighted asymmetric loss that combines multiple weights from label frequency, ground truth values, and relationships of labels.
The loss function is constructed with the aim of suppressing the influence of low-frequency labels that hinder meaningful learning of a model.
Experimental results showed that our method outperformed other methods especially on imbalanced datasets.
Moreover, we confirmed that our label smoothing method, which distributes the strength of relationships to the ground-truth values of related labels, contributes the most to improving the classification scores.
In future work, we will study combinations of more complex models with the proposed method.



% Start bibliography
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.7}
\bibliography{12refs}

% Start appendix
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\appendix
\section{Hyperparameter Settings for Each Method}
\label{sec:appendix-a}
In this section, we explain the hyperparameters for our main experiment and the additional experiment on 20 Newsgroups. 
Table \ref{tab:hyperparameters-main} lists the hyperparameters that were used in the main experiment on each dataset, and Table \ref{tab:hyperparameters-20} lists the hyperparameters that were used in the additional experiment on 20 Newsgroups.

\clearpage

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T10
\begin{table*}[t]
\input{12table10.tex}
  \caption{All hyperparameters for the main experiment.}
  \label{tab:hyperparameters-main}
\end{table*}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\clearpage

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T11
\begin{table*}[t]
\input{12table11.tex}
  \caption{hyperparameters for the 20 Newsgroups.}
  \label{tab:hyperparameters-20}
\end{table*}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



% Start biography
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Yuki Yasuda}{%
Yuki Yasuda received his B.H. and M.H. degrees in Human Science from Waseda University in 2015 and 2017, respectively. After his graduation, he joined NHK (Japan Broadcasting Corporation) and has been working at the Science and Technology Research Laboratories since 2019. 
}

\bioauthor[:]{Taro Miyazaki}{%
Taro Miyazaki received his B.E. and M.E. degrees in Information Science from Tokyo Institute of Technology in 2004 and 2006, respectively. After his graduation, he joined NHK (Japan Broadcasting Corporation) and has been working at the Science and Technology Research Laboratories since 2011. From 2017 to 2018, he was a visiting scholar at the University of \mbox{Melbourne}. His research interests include machine learning, data mining, and information retrieval. 
}

\bioauthor[:]{Jun Goto}{%
Jun Goto received his B.E. and M.E. degrees in electrical engineering from Tokushima University in 1991 and 1993, respectively. He joined NHK (Japan Broadcasting Corporation) in 1993 and has been with the Science and Technology Research Laboratories since 1998. He received his Ph.D. in informatics from the Graduate University for Advanced Study in 2014. His research interests include big data analysis and information extraction based on natural language processing.
}
\end{biography}

 \biodate   %%%% Following settings will be edited by editors.


\end{document}

